2024-12-02 17:14:19,227 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 17:14:19,256 main DEBUG Took 0.026519 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 17:14:19,256 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 17:14:19,257 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 17:14:19,258 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 17:14:19,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,278 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 17:14:19,297 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,300 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,306 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,307 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,308 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,311 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,314 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,321 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,325 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,329 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,334 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 17:14:19,337 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,337 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 17:14:19,342 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 17:14:19,344 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 17:14:19,346 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 17:14:19,347 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 17:14:19,348 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 17:14:19,350 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 17:14:19,380 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 17:14:19,384 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 17:14:19,391 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 17:14:19,391 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 17:14:19,392 main DEBUG createAppenders(={Console}) 2024-12-02 17:14:19,393 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-02 17:14:19,393 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-02 17:14:19,394 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-02 17:14:19,395 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 17:14:19,396 main DEBUG OutputStream closed 2024-12-02 17:14:19,401 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 17:14:19,401 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 17:14:19,402 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-02 17:14:19,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 17:14:19,536 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 17:14:19,537 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 17:14:19,538 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 17:14:19,539 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 17:14:19,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 17:14:19,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 17:14:19,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 17:14:19,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 17:14:19,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 17:14:19,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 17:14:19,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 17:14:19,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 17:14:19,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 17:14:19,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 17:14:19,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 17:14:19,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 17:14:19,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 17:14:19,550 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 17:14:19,551 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-02 17:14:19,551 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 17:14:19,552 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-02T17:14:19,924 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4 2024-12-02 17:14:19,928 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 17:14:19,929 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T17:14:19,944 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-02T17:14:19,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T17:14:19,994 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5, deleteOnExit=true 2024-12-02T17:14:19,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T17:14:19,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/test.cache.data in system properties and HBase conf 2024-12-02T17:14:19,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T17:14:20,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/hadoop.log.dir in system properties and HBase conf 2024-12-02T17:14:20,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T17:14:20,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T17:14:20,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T17:14:20,167 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T17:14:20,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T17:14:20,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T17:14:20,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T17:14:20,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T17:14:20,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T17:14:20,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T17:14:20,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T17:14:20,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T17:14:20,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T17:14:20,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T17:14:20,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/nfs.dump.dir in system properties and HBase conf 2024-12-02T17:14:20,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/java.io.tmpdir in system properties and HBase conf 2024-12-02T17:14:20,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T17:14:20,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T17:14:20,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T17:14:21,615 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T17:14:21,731 INFO [Time-limited test {}] log.Log(170): Logging initialized @3569ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T17:14:21,833 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:14:21,924 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T17:14:21,945 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T17:14:21,945 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T17:14:21,946 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T17:14:21,958 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:14:21,960 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/hadoop.log.dir/,AVAILABLE} 2024-12-02T17:14:21,961 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T17:14:22,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/java.io.tmpdir/jetty-localhost-41305-hadoop-hdfs-3_4_1-tests_jar-_-any-7306996572796576747/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T17:14:22,206 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41305} 2024-12-02T17:14:22,206 INFO [Time-limited test {}] server.Server(415): Started @4045ms 2024-12-02T17:14:22,946 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T17:14:22,970 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T17:14:22,979 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T17:14:22,979 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T17:14:22,980 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T17:14:22,981 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/hadoop.log.dir/,AVAILABLE} 2024-12-02T17:14:22,982 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47c7fd6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T17:14:23,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@250cb724{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/java.io.tmpdir/jetty-localhost-43705-hadoop-hdfs-3_4_1-tests_jar-_-any-13926890023403799805/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:14:23,180 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@69edd336{HTTP/1.1, (http/1.1)}{localhost:43705} 2024-12-02T17:14:23,181 INFO [Time-limited test {}] server.Server(415): Started @5020ms 2024-12-02T17:14:23,265 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T17:14:24,061 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/data/data1/current/BP-411259783-172.17.0.3-1733159661250/current, will proceed with Du for space computation calculation, 2024-12-02T17:14:24,073 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/data/data2/current/BP-411259783-172.17.0.3-1733159661250/current, will proceed with Du for space computation calculation, 2024-12-02T17:14:24,127 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T17:14:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda4883a2a9d4801c with lease ID 0x6b94be5fb65e15aa: Processing first storage report for DS-37ca2609-0b37-4f55-9344-89dd44a5fa16 from datanode DatanodeRegistration(127.0.0.1:36507, datanodeUuid=1be79f1d-8f15-4d6f-a492-629389f9b831, infoPort=39495, infoSecurePort=0, ipcPort=34031, storageInfo=lv=-57;cid=testClusterID;nsid=1691880308;c=1733159661250) 2024-12-02T17:14:24,205 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda4883a2a9d4801c with lease ID 0x6b94be5fb65e15aa: from storage DS-37ca2609-0b37-4f55-9344-89dd44a5fa16 node DatanodeRegistration(127.0.0.1:36507, datanodeUuid=1be79f1d-8f15-4d6f-a492-629389f9b831, infoPort=39495, infoSecurePort=0, ipcPort=34031, storageInfo=lv=-57;cid=testClusterID;nsid=1691880308;c=1733159661250), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-02T17:14:24,205 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda4883a2a9d4801c with lease ID 0x6b94be5fb65e15aa: Processing first storage report for DS-dae258a7-590c-474f-9e67-b45a238cb257 from datanode DatanodeRegistration(127.0.0.1:36507, datanodeUuid=1be79f1d-8f15-4d6f-a492-629389f9b831, infoPort=39495, infoSecurePort=0, ipcPort=34031, storageInfo=lv=-57;cid=testClusterID;nsid=1691880308;c=1733159661250) 2024-12-02T17:14:24,206 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda4883a2a9d4801c with lease ID 0x6b94be5fb65e15aa: from storage DS-dae258a7-590c-474f-9e67-b45a238cb257 node DatanodeRegistration(127.0.0.1:36507, datanodeUuid=1be79f1d-8f15-4d6f-a492-629389f9b831, infoPort=39495, infoSecurePort=0, ipcPort=34031, storageInfo=lv=-57;cid=testClusterID;nsid=1691880308;c=1733159661250), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T17:14:24,240 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4 2024-12-02T17:14:24,358 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/zookeeper_0, clientPort=58420, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T17:14:24,379 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58420 2024-12-02T17:14:24,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:24,410 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:24,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741825_1001 (size=7) 2024-12-02T17:14:24,859 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 with version=8 2024-12-02T17:14:24,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/hbase-staging 2024-12-02T17:14:25,032 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T17:14:25,426 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3b3375d1a925:0 server-side Connection retries=45 2024-12-02T17:14:25,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:14:25,441 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T17:14:25,448 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T17:14:25,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:14:25,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T17:14:25,654 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T17:14:25,758 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T17:14:25,772 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T17:14:25,777 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T17:14:25,815 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2780 (auto-detected) 2024-12-02T17:14:25,816 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-02T17:14:25,842 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41221 2024-12-02T17:14:25,871 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41221 connecting to ZooKeeper ensemble=127.0.0.1:58420 2024-12-02T17:14:25,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412210x0, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T17:14:25,942 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41221-0x1009bfbc5620000 connected 2024-12-02T17:14:26,001 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:26,004 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:26,035 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:14:26,040 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0, hbase.cluster.distributed=false 2024-12-02T17:14:26,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T17:14:26,078 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41221 2024-12-02T17:14:26,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41221 2024-12-02T17:14:26,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41221 2024-12-02T17:14:26,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41221 2024-12-02T17:14:26,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41221 2024-12-02T17:14:26,275 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3b3375d1a925:0 server-side Connection retries=45 2024-12-02T17:14:26,278 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:14:26,278 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T17:14:26,279 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T17:14:26,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T17:14:26,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T17:14:26,283 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T17:14:26,286 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T17:14:26,289 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40723 2024-12-02T17:14:26,291 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40723 connecting to ZooKeeper ensemble=127.0.0.1:58420 2024-12-02T17:14:26,298 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:26,302 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:26,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407230x0, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T17:14:26,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407230x0, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:14:26,341 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40723-0x1009bfbc5620001 connected 2024-12-02T17:14:26,341 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T17:14:26,355 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T17:14:26,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T17:14:26,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T17:14:26,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40723 2024-12-02T17:14:26,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40723 2024-12-02T17:14:26,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40723 2024-12-02T17:14:26,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40723 2024-12-02T17:14:26,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40723 2024-12-02T17:14:26,417 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3b3375d1a925:41221 2024-12-02T17:14:26,418 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3b3375d1a925,41221,1733159665139 2024-12-02T17:14:26,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:14:26,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:14:26,443 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3b3375d1a925,41221,1733159665139 2024-12-02T17:14:26,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:26,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T17:14:26,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:26,477 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T17:14:26,478 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3b3375d1a925,41221,1733159665139 from backup master directory 2024-12-02T17:14:26,483 WARN [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T17:14:26,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:14:26,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3b3375d1a925,41221,1733159665139 2024-12-02T17:14:26,484 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3b3375d1a925,41221,1733159665139 2024-12-02T17:14:26,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T17:14:26,487 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T17:14:26,489 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T17:14:26,595 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/hbase.id] with ID: dd8a279c-f176-4ff6-92c2-4858730f937a 2024-12-02T17:14:26,595 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/.tmp/hbase.id 2024-12-02T17:14:26,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741826_1002 (size=42) 2024-12-02T17:14:27,028 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/.tmp/hbase.id]:[hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/hbase.id] 2024-12-02T17:14:27,079 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T17:14:27,085 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T17:14:27,112 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-12-02T17:14:27,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:27,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:27,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741827_1003 (size=196) 2024-12-02T17:14:27,176 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:14:27,179 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T17:14:27,209 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:27,216 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:14:27,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741828_1004 (size=1189) 2024-12-02T17:14:27,698 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store 2024-12-02T17:14:27,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741829_1005 (size=34) 2024-12-02T17:14:27,735 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T17:14:27,739 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:14:27,740 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T17:14:27,740 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:14:27,741 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:14:27,742 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T17:14:27,742 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:14:27,743 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:14:27,744 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733159667740Disabling compacts and flushes for region at 1733159667740Disabling writes for close at 1733159667742 (+2 ms)Writing region close event to WAL at 1733159667743 (+1 ms)Closed at 1733159667743 2024-12-02T17:14:27,747 WARN [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/.initializing 2024-12-02T17:14:27,747 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/WALs/3b3375d1a925,41221,1733159665139 2024-12-02T17:14:27,759 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T17:14:27,781 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C41221%2C1733159665139, suffix=, logDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/WALs/3b3375d1a925,41221,1733159665139, archiveDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/oldWALs, maxLogs=10 2024-12-02T17:14:27,808 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/WALs/3b3375d1a925,41221,1733159665139/3b3375d1a925%2C41221%2C1733159665139.1733159667786, exclude list is [], retry=0 2024-12-02T17:14:27,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36507,DS-37ca2609-0b37-4f55-9344-89dd44a5fa16,DISK] 2024-12-02T17:14:27,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T17:14:27,887 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/WALs/3b3375d1a925,41221,1733159665139/3b3375d1a925%2C41221%2C1733159665139.1733159667786 2024-12-02T17:14:27,888 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39495:39495)] 2024-12-02T17:14:27,889 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:14:27,890 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:14:27,894 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:27,897 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:27,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:27,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T17:14:27,998 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:28,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T17:14:28,009 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:14:28,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T17:14:28,019 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:14:28,021 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,025 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T17:14:28,026 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:14:28,027 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,033 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,035 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,043 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,044 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,051 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T17:14:28,059 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T17:14:28,070 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:14:28,073 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73048549, jitterRate=0.08850820362567902}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T17:14:28,082 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733159667913Initializing all the Stores at 1733159667918 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159667919 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159667921 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159667921Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159667921Cleaning up temporary data from old regions at 1733159668044 (+123 ms)Region opened successfully at 1733159668082 (+38 ms) 2024-12-02T17:14:28,084 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T17:14:28,127 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c580f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b3375d1a925/172.17.0.3:0 2024-12-02T17:14:28,172 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T17:14:28,188 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T17:14:28,188 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T17:14:28,192 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T17:14:28,194 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T17:14:28,200 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-02T17:14:28,200 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T17:14:28,241 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T17:14:28,256 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T17:14:28,260 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T17:14:28,263 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T17:14:28,267 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T17:14:28,269 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T17:14:28,272 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T17:14:28,275 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T17:14:28,277 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T17:14:28,279 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T17:14:28,281 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T17:14:28,305 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T17:14:28,308 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T17:14:28,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T17:14:28,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:28,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T17:14:28,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:28,320 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3b3375d1a925,41221,1733159665139, sessionid=0x1009bfbc5620000, setting cluster-up flag (Was=false) 2024-12-02T17:14:28,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:28,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:28,344 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T17:14:28,348 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3b3375d1a925,41221,1733159665139 2024-12-02T17:14:28,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:28,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:28,362 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T17:14:28,364 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3b3375d1a925,41221,1733159665139 2024-12-02T17:14:28,372 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T17:14:28,406 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(746): ClusterId : dd8a279c-f176-4ff6-92c2-4858730f937a 2024-12-02T17:14:28,409 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T17:14:28,415 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T17:14:28,415 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T17:14:28,418 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T17:14:28,418 DEBUG [RS:0;3b3375d1a925:40723 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd86ae9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3b3375d1a925/172.17.0.3:0 2024-12-02T17:14:28,438 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3b3375d1a925:40723 2024-12-02T17:14:28,441 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T17:14:28,441 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T17:14:28,442 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T17:14:28,445 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,41221,1733159665139 with port=40723, startcode=1733159666211 2024-12-02T17:14:28,460 DEBUG [RS:0;3b3375d1a925:40723 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T17:14:28,463 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T17:14:28,474 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T17:14:28,482 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T17:14:28,490 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3b3375d1a925,41221,1733159665139 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T17:14:28,498 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38821, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T17:14:28,505 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41221 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:28,507 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:14:28,507 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:14:28,508 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:14:28,508 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3b3375d1a925:0, corePoolSize=5, maxPoolSize=5 2024-12-02T17:14:28,508 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3b3375d1a925:0, corePoolSize=10, maxPoolSize=10 2024-12-02T17:14:28,509 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,510 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3b3375d1a925:0, corePoolSize=2, maxPoolSize=2 2024-12-02T17:14:28,510 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,523 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733159698523 2024-12-02T17:14:28,523 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T17:14:28,524 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T17:14:28,525 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T17:14:28,526 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T17:14:28,541 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,542 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-02T17:14:28,542 WARN [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-02T17:14:28,542 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T17:14:28,542 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T17:14:28,543 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T17:14:28,543 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T17:14:28,543 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T17:14:28,544 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,549 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T17:14:28,550 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T17:14:28,550 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T17:14:28,553 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T17:14:28,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741831_1007 (size=1321) 2024-12-02T17:14:28,553 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T17:14:28,555 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.large.0-1733159668555,5,FailOnTimeoutGroup] 2024-12-02T17:14:28,565 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T17:14:28,565 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:14:28,568 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.small.0-1733159668556,5,FailOnTimeoutGroup] 2024-12-02T17:14:28,568 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,569 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T17:14:28,570 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,570 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741832_1008 (size=32) 2024-12-02T17:14:28,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:14:28,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T17:14:28,615 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T17:14:28,615 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,616 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:28,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T17:14:28,620 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T17:14:28,620 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:28,621 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T17:14:28,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T17:14:28,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:28,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T17:14:28,628 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T17:14:28,628 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:28,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:28,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T17:14:28,634 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740 2024-12-02T17:14:28,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740 2024-12-02T17:14:28,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T17:14:28,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T17:14:28,642 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:14:28,643 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(2659): reportForDuty to master=3b3375d1a925,41221,1733159665139 with port=40723, startcode=1733159666211 2024-12-02T17:14:28,646 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41221 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3b3375d1a925,40723,1733159666211 2024-12-02T17:14:28,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T17:14:28,649 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41221 {}] master.ServerManager(517): Registering regionserver=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:28,660 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:14:28,660 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:14:28,660 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33791 2024-12-02T17:14:28,660 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T17:14:28,661 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59357342, jitterRate=-0.11550667881965637}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:14:28,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733159668604Initializing all the Stores at 1733159668607 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159668607Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159668607Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159668607Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159668607Cleaning up temporary data from old regions at 1733159668640 (+33 ms)Region opened successfully at 1733159668664 (+24 ms) 2024-12-02T17:14:28,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T17:14:28,665 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T17:14:28,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T17:14:28,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T17:14:28,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T17:14:28,669 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T17:14:28,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733159668665Disabling compacts and flushes for region at 1733159668665Disabling writes for close at 1733159668665Writing region close event to WAL at 1733159668669 (+4 ms)Closed at 1733159668669 2024-12-02T17:14:28,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T17:14:28,672 DEBUG [RS:0;3b3375d1a925:40723 {}] zookeeper.ZKUtil(111): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3b3375d1a925,40723,1733159666211 2024-12-02T17:14:28,672 WARN [RS:0;3b3375d1a925:40723 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T17:14:28,672 INFO [RS:0;3b3375d1a925:40723 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:14:28,673 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211 2024-12-02T17:14:28,674 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T17:14:28,674 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T17:14:28,676 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3b3375d1a925,40723,1733159666211] 2024-12-02T17:14:28,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T17:14:28,694 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:14:28,699 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T17:14:28,729 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T17:14:28,760 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T17:14:28,773 INFO [RS:0;3b3375d1a925:40723 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T17:14:28,773 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,777 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T17:14:28,787 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T17:14:28,788 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,789 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,789 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,789 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,790 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,790 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,790 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3b3375d1a925:0, corePoolSize=2, maxPoolSize=2 2024-12-02T17:14:28,790 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,791 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,791 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,791 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,791 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,792 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3b3375d1a925:0, corePoolSize=1, maxPoolSize=1 2024-12-02T17:14:28,792 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:14:28,792 DEBUG [RS:0;3b3375d1a925:40723 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0, corePoolSize=3, maxPoolSize=3 2024-12-02T17:14:28,805 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,805 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,806 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,806 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,807 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,807 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,40723,1733159666211-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T17:14:28,839 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T17:14:28,842 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,40723,1733159666211-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,842 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,842 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.Replication(171): 3b3375d1a925,40723,1733159666211 started 2024-12-02T17:14:28,851 WARN [3b3375d1a925:41221 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T17:14:28,874 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:28,875 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1482): Serving as 3b3375d1a925,40723,1733159666211, RpcServer on 3b3375d1a925/172.17.0.3:40723, sessionid=0x1009bfbc5620001 2024-12-02T17:14:28,876 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T17:14:28,876 DEBUG [RS:0;3b3375d1a925:40723 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3b3375d1a925,40723,1733159666211 2024-12-02T17:14:28,876 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,40723,1733159666211' 2024-12-02T17:14:28,877 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T17:14:28,885 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T17:14:28,894 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T17:14:28,894 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T17:14:28,895 DEBUG [RS:0;3b3375d1a925:40723 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3b3375d1a925,40723,1733159666211 2024-12-02T17:14:28,899 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3b3375d1a925,40723,1733159666211' 2024-12-02T17:14:28,899 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T17:14:28,913 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T17:14:28,914 DEBUG [RS:0;3b3375d1a925:40723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T17:14:28,914 INFO [RS:0;3b3375d1a925:40723 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T17:14:28,914 INFO [RS:0;3b3375d1a925:40723 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T17:14:29,021 INFO [RS:0;3b3375d1a925:40723 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T17:14:29,025 INFO [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C40723%2C1733159666211, suffix=, logDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211, archiveDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/oldWALs, maxLogs=32 2024-12-02T17:14:29,061 DEBUG [RS:0;3b3375d1a925:40723 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211/3b3375d1a925%2C40723%2C1733159666211.1733159669029, exclude list is [], retry=0 2024-12-02T17:14:29,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36507,DS-37ca2609-0b37-4f55-9344-89dd44a5fa16,DISK] 2024-12-02T17:14:29,080 INFO [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211/3b3375d1a925%2C40723%2C1733159666211.1733159669029 2024-12-02T17:14:29,081 DEBUG [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39495:39495)] 2024-12-02T17:14:29,104 DEBUG [3b3375d1a925:41221 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T17:14:29,118 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:29,128 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,40723,1733159666211, state=OPENING 2024-12-02T17:14:29,134 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T17:14:29,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:29,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:14:29,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:14:29,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:14:29,145 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T17:14:29,148 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:14:29,334 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T17:14:29,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T17:14:29,370 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T17:14:29,370 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T17:14:29,374 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T17:14:29,382 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3b3375d1a925%2C40723%2C1733159666211.meta, suffix=.meta, logDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211, archiveDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/oldWALs, maxLogs=32 2024-12-02T17:14:29,406 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211/3b3375d1a925%2C40723%2C1733159666211.meta.1733159669385.meta, exclude list is [], retry=0 2024-12-02T17:14:29,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36507,DS-37ca2609-0b37-4f55-9344-89dd44a5fa16,DISK] 2024-12-02T17:14:29,418 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/WALs/3b3375d1a925,40723,1733159666211/3b3375d1a925%2C40723%2C1733159666211.meta.1733159669385.meta 2024-12-02T17:14:29,425 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39495:39495)] 2024-12-02T17:14:29,425 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:14:29,428 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T17:14:29,431 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T17:14:29,438 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T17:14:29,443 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T17:14:29,444 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:14:29,444 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T17:14:29,445 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T17:14:29,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T17:14:29,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T17:14:29,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:29,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:29,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T17:14:29,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T17:14:29,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:29,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:29,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T17:14:29,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T17:14:29,464 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:29,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:29,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T17:14:29,466 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T17:14:29,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:29,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T17:14:29,468 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T17:14:29,470 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740 2024-12-02T17:14:29,473 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740 2024-12-02T17:14:29,479 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T17:14:29,479 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T17:14:29,480 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:14:29,489 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T17:14:29,496 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69939762, jitterRate=0.04218366742134094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:14:29,497 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T17:14:29,499 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733159669445Writing region info on filesystem at 1733159669445Initializing all the Stores at 1733159669449 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159669449Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159669449Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159669449Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733159669449Cleaning up temporary data from old regions at 1733159669479 (+30 ms)Running coprocessor post-open hooks at 1733159669497 (+18 ms)Region opened successfully at 1733159669498 (+1 ms) 2024-12-02T17:14:29,517 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733159669322 2024-12-02T17:14:29,538 DEBUG [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T17:14:29,539 INFO [RS_OPEN_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T17:14:29,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:29,546 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3b3375d1a925,40723,1733159666211, state=OPEN 2024-12-02T17:14:29,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:14:29,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T17:14:29,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:14:29,561 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:29,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T17:14:29,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T17:14:29,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3b3375d1a925,40723,1733159666211 in 413 msec 2024-12-02T17:14:29,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T17:14:29,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 892 msec 2024-12-02T17:14:29,585 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T17:14:29,585 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T17:14:29,614 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:29,616 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:29,649 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:29,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:29,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2820 sec 2024-12-02T17:14:29,692 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733159669692, completionTime=-1 2024-12-02T17:14:29,695 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T17:14:29,695 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T17:14:29,729 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T17:14:29,729 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733159729729 2024-12-02T17:14:29,729 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733159789729 2024-12-02T17:14:29,729 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-02T17:14:29,732 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,41221,1733159665139-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,732 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,41221,1733159665139-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,732 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,41221,1733159665139-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,734 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3b3375d1a925:41221, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,737 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,738 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,742 DEBUG [master/3b3375d1a925:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T17:14:29,766 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.282sec 2024-12-02T17:14:29,767 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T17:14:29,769 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T17:14:29,770 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T17:14:29,770 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T17:14:29,771 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T17:14:29,772 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,41221,1733159665139-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T17:14:29,772 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,41221,1733159665139-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T17:14:29,782 DEBUG [master/3b3375d1a925:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T17:14:29,783 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T17:14:29,783 INFO [master/3b3375d1a925:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3b3375d1a925,41221,1733159665139-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T17:14:29,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:29,821 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T17:14:29,821 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T17:14:29,824 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:29,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:29,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:29,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40f7a663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:29,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:29,849 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:29,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:29,853 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:29,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:29,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3b3375d1a925,41221,1733159665139 2024-12-02T17:14:29,928 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=855, ProcessCount=11, AvailableMemoryMB=4076 2024-12-02T17:14:29,939 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:14:29,953 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:14:29,956 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3fcbafaf 2024-12-02T17:14:29,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:14:29,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:14:29,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:14:29,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:14:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T17:14:30,002 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:14:30,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-02T17:14:30,005 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:14:30,013 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:14:30,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741835_1011 (size=1061) 2024-12-02T17:14:30,033 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:14:30,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741836_1012 (size=53) 2024-12-02T17:14:30,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:14:30,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dcecfb16214ccdbe1c27643e3ed0b42c, disabling compactions & flushes 2024-12-02T17:14:30,061 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. after waiting 0 ms 2024-12-02T17:14:30,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,061 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,061 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dcecfb16214ccdbe1c27643e3ed0b42c: Waiting for close lock at 1733159670060Disabling compacts and flushes for region at 1733159670060Disabling writes for close at 1733159670061 (+1 ms)Writing region close event to WAL at 1733159670061Closed at 1733159670061 2024-12-02T17:14:30,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:14:30,075 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733159670067"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159670067"}]},"ts":"1733159670067"} 2024-12-02T17:14:30,081 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T17:14:30,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:14:30,086 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159670083"}]},"ts":"1733159670083"} 2024-12-02T17:14:30,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T17:14:30,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, ASSIGN}] 2024-12-02T17:14:30,099 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, ASSIGN 2024-12-02T17:14:30,101 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, ASSIGN; state=OFFLINE, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=false 2024-12-02T17:14:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:14:30,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dcecfb16214ccdbe1c27643e3ed0b42c, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:30,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, ASSIGN because future has completed 2024-12-02T17:14:30,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:14:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:14:30,423 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,424 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:14:30,424 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,425 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:14:30,425 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,425 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,428 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,431 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:14:30,432 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcecfb16214ccdbe1c27643e3ed0b42c columnFamilyName A 2024-12-02T17:14:30,432 DEBUG [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:30,433 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.HStore(327): Store=dcecfb16214ccdbe1c27643e3ed0b42c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:14:30,434 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,436 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:14:30,436 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcecfb16214ccdbe1c27643e3ed0b42c columnFamilyName B 2024-12-02T17:14:30,436 DEBUG [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:30,437 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.HStore(327): Store=dcecfb16214ccdbe1c27643e3ed0b42c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:14:30,437 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,439 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:14:30,439 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcecfb16214ccdbe1c27643e3ed0b42c columnFamilyName C 2024-12-02T17:14:30,439 DEBUG [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:14:30,440 INFO [StoreOpener-dcecfb16214ccdbe1c27643e3ed0b42c-1 {}] regionserver.HStore(327): Store=dcecfb16214ccdbe1c27643e3ed0b42c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:14:30,441 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,441 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,442 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,443 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,445 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,445 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,447 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:14:30,450 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,457 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:14:30,459 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dcecfb16214ccdbe1c27643e3ed0b42c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64433010, jitterRate=-0.039873331785202026}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:14:30,459 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,460 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dcecfb16214ccdbe1c27643e3ed0b42c: Running coprocessor pre-open hook at 1733159670425Writing region info on filesystem at 1733159670425Initializing all the Stores at 1733159670427 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159670427Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159670427Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159670427Cleaning up temporary data from old regions at 1733159670446 (+19 ms)Running coprocessor post-open hooks at 1733159670459 (+13 ms)Region opened successfully at 1733159670460 (+1 ms) 2024-12-02T17:14:30,462 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., pid=6, masterSystemTime=1733159670416 2024-12-02T17:14:30,467 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,467 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,469 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dcecfb16214ccdbe1c27643e3ed0b42c, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:30,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:14:30,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T17:14:30,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 in 216 msec 2024-12-02T17:14:30,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T17:14:30,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, ASSIGN in 388 msec 2024-12-02T17:14:30,488 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:14:30,489 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159670488"}]},"ts":"1733159670488"} 2024-12-02T17:14:30,492 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T17:14:30,493 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:14:30,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 509 msec 2024-12-02T17:14:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T17:14:30,638 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:30,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205635fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,645 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60daaef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,647 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,649 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bebe1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,650 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,657 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c7f33c8 2024-12-02T17:14:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:14:30,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:14:30,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,674 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,679 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d3010f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3280f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,703 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71435d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686b49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b90827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,712 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45584418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8cc673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,717 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f11d0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b92dc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e539128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:14:30,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:14:30,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:14:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:14:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:14:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:30,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:14:30,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c944594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:14:30,735 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,735 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,743 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,743 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,748 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,764 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,764 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:30,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,766 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-02T17:14:30,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:30,775 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,777 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:30,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:30,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,783 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:14:30,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:14:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:14:30,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:14:30,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:14:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:30,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:14:30,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:30,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:30,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:30,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:30,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:30,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:30,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:30,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:30,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/c4dc4a0d2019472ca087aae7afe76ee4 is 50, key is test_row_0/A:col10/1733159670860/Put/seqid=0 2024-12-02T17:14:31,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741837_1013 (size=12001) 2024-12-02T17:14:31,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/c4dc4a0d2019472ca087aae7afe76ee4 2024-12-02T17:14:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:31,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159731083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159731086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159731087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159731099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159731096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:31,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2aa30f07eaba49cea47a02815ab4185c is 50, key is test_row_0/B:col10/1733159670860/Put/seqid=0 2024-12-02T17:14:31,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159731249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741838_1014 (size=12001) 2024-12-02T17:14:31,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159731249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159731250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159731251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159731251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:31,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:31,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159731468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159731468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159731469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159731469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159731482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:31,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:31,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:31,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:31,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2aa30f07eaba49cea47a02815ab4185c 2024-12-02T17:14:31,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/241b3c4839e643ba961017a9595d842c is 50, key is test_row_0/C:col10/1733159670860/Put/seqid=0 2024-12-02T17:14:31,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741839_1015 (size=12001) 2024-12-02T17:14:31,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/241b3c4839e643ba961017a9595d842c 2024-12-02T17:14:31,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/c4dc4a0d2019472ca087aae7afe76ee4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/c4dc4a0d2019472ca087aae7afe76ee4 2024-12-02T17:14:31,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/c4dc4a0d2019472ca087aae7afe76ee4, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T17:14:31,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2aa30f07eaba49cea47a02815ab4185c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2aa30f07eaba49cea47a02815ab4185c 2024-12-02T17:14:31,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2aa30f07eaba49cea47a02815ab4185c, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T17:14:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/241b3c4839e643ba961017a9595d842c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/241b3c4839e643ba961017a9595d842c 2024-12-02T17:14:31,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/241b3c4839e643ba961017a9595d842c, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T17:14:31,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dcecfb16214ccdbe1c27643e3ed0b42c in 972ms, sequenceid=13, compaction requested=false 2024-12-02T17:14:31,839 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T17:14:31,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:31,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T17:14:31,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:31,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:31,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:31,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:31,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ac911b2598514bf89bc27328464f9626 is 50, key is test_row_0/A:col10/1733159671858/Put/seqid=0 2024-12-02T17:14:31,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:31,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:31,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:31,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741840_1016 (size=12001) 2024-12-02T17:14:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:31,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ac911b2598514bf89bc27328464f9626 2024-12-02T17:14:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159731966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159731981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/4423e45c383f485a8b87f79076c7de7d is 50, key is test_row_0/B:col10/1733159671858/Put/seqid=0 2024-12-02T17:14:32,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159731983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159731986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159731996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741841_1017 (size=12001) 2024-12-02T17:14:32,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159732113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:32,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159732129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159732148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159732148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159732149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:32,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159732337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159732369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159732369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159732369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159732370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/4423e45c383f485a8b87f79076c7de7d 2024-12-02T17:14:32,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/5855d074785f4d268efeb00fb2faaf32 is 50, key is test_row_0/C:col10/1733159671858/Put/seqid=0 2024-12-02T17:14:32,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:32,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741842_1018 (size=12001) 2024-12-02T17:14:32,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/5855d074785f4d268efeb00fb2faaf32 2024-12-02T17:14:32,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ac911b2598514bf89bc27328464f9626 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ac911b2598514bf89bc27328464f9626 2024-12-02T17:14:32,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ac911b2598514bf89bc27328464f9626, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T17:14:32,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/4423e45c383f485a8b87f79076c7de7d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/4423e45c383f485a8b87f79076c7de7d 2024-12-02T17:14:32,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/4423e45c383f485a8b87f79076c7de7d, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T17:14:32,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/5855d074785f4d268efeb00fb2faaf32 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/5855d074785f4d268efeb00fb2faaf32 2024-12-02T17:14:32,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/5855d074785f4d268efeb00fb2faaf32, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T17:14:32,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=46.96 KB/48090 for dcecfb16214ccdbe1c27643e3ed0b42c in 721ms, sequenceid=36, compaction requested=false 2024-12-02T17:14:32,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:32,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T17:14:32,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:32,650 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:14:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:32,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ae3d863d577342bf80ad575ab6415a21 is 50, key is test_row_0/A:col10/1733159671976/Put/seqid=0 2024-12-02T17:14:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:32,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:32,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741843_1019 (size=12001) 2024-12-02T17:14:32,685 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ae3d863d577342bf80ad575ab6415a21 2024-12-02T17:14:32,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/07ffc256e7f94541a436d0bd93275e02 is 50, key is test_row_0/B:col10/1733159671976/Put/seqid=0 2024-12-02T17:14:32,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741844_1020 (size=12001) 2024-12-02T17:14:32,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159672728, max timestamp=1733159672766Num uniques 859; 2024-12-02T17:14:32,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159672728, max timestamp=1733159672766Num uniques 859; 2024-12-02T17:14:32,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:32,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:32,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159672728, max timestamp=1733159672766Num uniques 859; 2024-12-02T17:14:32,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:32,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159732855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159732858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159732847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159732872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159732872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:32,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159732980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159732987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159732988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:32,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159732997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159732997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/07ffc256e7f94541a436d0bd93275e02 2024-12-02T17:14:33,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1b823bdc0e7e46ae8ae782666504d8b7 is 50, key is test_row_0/C:col10/1733159671976/Put/seqid=0 2024-12-02T17:14:33,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741845_1021 (size=12001) 2024-12-02T17:14:33,207 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1b823bdc0e7e46ae8ae782666504d8b7 2024-12-02T17:14:33,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159733197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159733207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159733207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ae3d863d577342bf80ad575ab6415a21 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae3d863d577342bf80ad575ab6415a21 2024-12-02T17:14:33,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159733217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159733217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,232 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae3d863d577342bf80ad575ab6415a21, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T17:14:33,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/07ffc256e7f94541a436d0bd93275e02 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/07ffc256e7f94541a436d0bd93275e02 2024-12-02T17:14:33,251 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/07ffc256e7f94541a436d0bd93275e02, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T17:14:33,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1b823bdc0e7e46ae8ae782666504d8b7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1b823bdc0e7e46ae8ae782666504d8b7 2024-12-02T17:14:33,279 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1b823bdc0e7e46ae8ae782666504d8b7, entries=150, sequenceid=46, filesize=11.7 K 2024-12-02T17:14:33,281 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dcecfb16214ccdbe1c27643e3ed0b42c in 631ms, sequenceid=46, compaction requested=true 2024-12-02T17:14:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T17:14:33,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T17:14:33,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T17:14:33,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5050 sec 2024-12-02T17:14:33,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5250 sec 2024-12-02T17:14:33,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159672842, max timestamp=1733159673531Num uniques 1018; 2024-12-02T17:14:33,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T17:14:33,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=116.29 KB heapSize=332.05 KB 2024-12-02T17:14:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:33,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159672842, max timestamp=1733159673531Num uniques 1018; 2024-12-02T17:14:33,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T17:14:33,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7684, totalHeapSize=17712, min timestamp=1733159672842, max timestamp=1733159673536Num uniques 1018; 2024-12-02T17:14:33,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46614, new segment data size=7684, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T17:14:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:33,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:33,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:33,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:33,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:33,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:33,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/eac15d4ba601488381aaf374c61c387b is 50, key is test_row_0/A:col10/1733159673536/Put/seqid=0 2024-12-02T17:14:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741846_1022 (size=12001) 2024-12-02T17:14:33,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/eac15d4ba601488381aaf374c61c387b 2024-12-02T17:14:33,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733159673574, max timestamp=1733159673611Num uniques 859; 2024-12-02T17:14:33,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-02T17:14:33,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1733159673574, max timestamp=1733159673611Num uniques 859; 2024-12-02T17:14:33,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-12-02T17:14:33,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159673591, max timestamp=1733159673611Num uniques 859; 2024-12-02T17:14:33,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:33,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2237d6fcb1194d2d825f420b00d0ee4f is 50, key is test_row_0/B:col10/1733159673536/Put/seqid=0 2024-12-02T17:14:33,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159733678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159733678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159733679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159733691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159733692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741847_1023 (size=12001) 2024-12-02T17:14:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159733807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159733807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159733808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159733808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:33,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159733818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159734027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159734027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159734027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159734037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159734037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2237d6fcb1194d2d825f420b00d0ee4f 2024-12-02T17:14:34,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/6213b109cee94c47867052ee084905bd is 50, key is test_row_0/C:col10/1733159673536/Put/seqid=0 2024-12-02T17:14:34,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741848_1024 (size=12001) 2024-12-02T17:14:34,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/6213b109cee94c47867052ee084905bd 2024-12-02T17:14:34,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/eac15d4ba601488381aaf374c61c387b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/eac15d4ba601488381aaf374c61c387b 2024-12-02T17:14:34,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/eac15d4ba601488381aaf374c61c387b, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T17:14:34,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2237d6fcb1194d2d825f420b00d0ee4f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2237d6fcb1194d2d825f420b00d0ee4f 2024-12-02T17:14:34,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2237d6fcb1194d2d825f420b00d0ee4f, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T17:14:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/6213b109cee94c47867052ee084905bd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/6213b109cee94c47867052ee084905bd 2024-12-02T17:14:34,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/6213b109cee94c47867052ee084905bd, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T17:14:34,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=152.07 KB/155720 for dcecfb16214ccdbe1c27643e3ed0b42c in 784ms, sequenceid=86, compaction requested=true 2024-12-02T17:14:34,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:34,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:34,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:34,354 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:34,354 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:34,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=158.78 KB heapSize=436.16 KB 2024-12-02T17:14:34,365 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:34,368 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:34,369 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:34,369 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2aa30f07eaba49cea47a02815ab4185c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/4423e45c383f485a8b87f79076c7de7d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/07ffc256e7f94541a436d0bd93275e02, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2237d6fcb1194d2d825f420b00d0ee4f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=46.9 K 2024-12-02T17:14:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:34,373 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa30f07eaba49cea47a02815ab4185c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733159670819 2024-12-02T17:14:34,373 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:34,374 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4423e45c383f485a8b87f79076c7de7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733159671858 2024-12-02T17:14:34,379 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:34,379 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:34,381 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/c4dc4a0d2019472ca087aae7afe76ee4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ac911b2598514bf89bc27328464f9626, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae3d863d577342bf80ad575ab6415a21, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/eac15d4ba601488381aaf374c61c387b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=46.9 K 2024-12-02T17:14:34,382 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 07ffc256e7f94541a436d0bd93275e02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733159671965 2024-12-02T17:14:34,382 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4dc4a0d2019472ca087aae7afe76ee4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733159670819 2024-12-02T17:14:34,385 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac911b2598514bf89bc27328464f9626, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733159671858 2024-12-02T17:14:34,387 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2237d6fcb1194d2d825f420b00d0ee4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159672868 2024-12-02T17:14:34,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159734367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159734378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159734385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,395 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3d863d577342bf80ad575ab6415a21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733159671965 2024-12-02T17:14:34,403 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac15d4ba601488381aaf374c61c387b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159672868 2024-12-02T17:14:34,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/14a685c335034110b01d4f9c82a5dd3e is 50, key is test_row_0/A:col10/1733159673685/Put/seqid=0 2024-12-02T17:14:34,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159734391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741849_1025 (size=12001) 2024-12-02T17:14:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159734391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/14a685c335034110b01d4f9c82a5dd3e 2024-12-02T17:14:34,501 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#13 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:34,503 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/ff0bf45030fb4cf8b3a523c1f991dcad is 50, key is test_row_0/B:col10/1733159673536/Put/seqid=0 2024-12-02T17:14:34,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159734498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,514 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#14 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:34,515 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/35c373df4701459cbc9cd775916a4886 is 50, key is test_row_0/A:col10/1733159673536/Put/seqid=0 2024-12-02T17:14:34,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159734499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/69f13fd2752847bfad9e80c04d775960 is 50, key is test_row_0/B:col10/1733159673685/Put/seqid=0 2024-12-02T17:14:34,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741850_1026 (size=12139) 2024-12-02T17:14:34,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159734563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741851_1027 (size=12139) 2024-12-02T17:14:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159734564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741852_1028 (size=12001) 2024-12-02T17:14:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159734718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159734737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159734777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159734783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159734897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:34,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:34,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:14:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T17:14:34,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:34,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-02T17:14:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T17:14:34,973 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:34,976 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:34,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:35,017 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/ff0bf45030fb4cf8b3a523c1f991dcad as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/ff0bf45030fb4cf8b3a523c1f991dcad 2024-12-02T17:14:35,017 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/35c373df4701459cbc9cd775916a4886 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/35c373df4701459cbc9cd775916a4886 2024-12-02T17:14:35,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/69f13fd2752847bfad9e80c04d775960 2024-12-02T17:14:35,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:35,060 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into ff0bf45030fb4cf8b3a523c1f991dcad(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:35,060 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159735047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:35,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159735048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:35,063 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=12, startTime=1733159674353; duration=0sec 2024-12-02T17:14:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:35,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/027765d4429348ee9a8cbd691d29c79e is 50, key is test_row_0/C:col10/1733159673685/Put/seqid=0 2024-12-02T17:14:35,064 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 35c373df4701459cbc9cd775916a4886(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:35,070 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:35,070 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=12, startTime=1733159674332; duration=0sec 2024-12-02T17:14:35,070 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:35,071 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:35,071 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:35,072 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:35,072 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T17:14:35,083 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:35,083 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:35,083 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:35,084 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/241b3c4839e643ba961017a9595d842c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/5855d074785f4d268efeb00fb2faaf32, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1b823bdc0e7e46ae8ae782666504d8b7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/6213b109cee94c47867052ee084905bd] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=46.9 K 2024-12-02T17:14:35,085 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 241b3c4839e643ba961017a9595d842c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733159670819 2024-12-02T17:14:35,091 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5855d074785f4d268efeb00fb2faaf32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733159671858 2024-12-02T17:14:35,093 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T17:14:35,095 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b823bdc0e7e46ae8ae782666504d8b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733159671965 2024-12-02T17:14:35,097 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6213b109cee94c47867052ee084905bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159672868 2024-12-02T17:14:35,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159735089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159735098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:35,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741853_1029 (size=12001) 2024-12-02T17:14:35,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/027765d4429348ee9a8cbd691d29c79e 2024-12-02T17:14:35,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T17:14:35,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:35,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:35,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:35,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:35,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/14a685c335034110b01d4f9c82a5dd3e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/14a685c335034110b01d4f9c82a5dd3e 2024-12-02T17:14:35,156 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#17 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:35,157 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1fd42dd3ca32462db6ff94c41a3c3b7b is 50, key is test_row_0/C:col10/1733159673536/Put/seqid=0 2024-12-02T17:14:35,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/14a685c335034110b01d4f9c82a5dd3e, entries=150, sequenceid=123, filesize=11.7 K 2024-12-02T17:14:35,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/69f13fd2752847bfad9e80c04d775960 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/69f13fd2752847bfad9e80c04d775960 2024-12-02T17:14:35,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/69f13fd2752847bfad9e80c04d775960, entries=150, sequenceid=123, filesize=11.7 K 2024-12-02T17:14:35,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/027765d4429348ee9a8cbd691d29c79e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/027765d4429348ee9a8cbd691d29c79e 2024-12-02T17:14:35,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741854_1030 (size=12139) 2024-12-02T17:14:35,240 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1fd42dd3ca32462db6ff94c41a3c3b7b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fd42dd3ca32462db6ff94c41a3c3b7b 2024-12-02T17:14:35,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/027765d4429348ee9a8cbd691d29c79e, entries=150, sequenceid=123, filesize=11.7 K 2024-12-02T17:14:35,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~165.49 KB/169460, heapSize ~456.03 KB/466976, currentSize=20.13 KB/20610 for dcecfb16214ccdbe1c27643e3ed0b42c in 893ms, sequenceid=123, compaction requested=false 2024-12-02T17:14:35,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:35,274 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 1fd42dd3ca32462db6ff94c41a3c3b7b(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:35,274 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:35,274 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=12, startTime=1733159674353; duration=0sec 2024-12-02T17:14:35,274 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:35,274 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T17:14:35,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T17:14:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:35,294 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-02T17:14:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:35,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/e37d94306ca3487eb0f966e607e3628f is 50, key is test_row_0/A:col10/1733159674385/Put/seqid=0 2024-12-02T17:14:35,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741855_1031 (size=9657) 2024-12-02T17:14:35,373 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/e37d94306ca3487eb0f966e607e3628f 2024-12-02T17:14:35,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/de7fde781b784f239f071c49b505457c is 50, key is test_row_0/B:col10/1733159674385/Put/seqid=0 2024-12-02T17:14:35,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741856_1032 (size=9657) 2024-12-02T17:14:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T17:14:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:35,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159675667, max timestamp=1733159675684Num uniques 859; 2024-12-02T17:14:35,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159675667, max timestamp=1733159675684Num uniques 859; 2024-12-02T17:14:35,693 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:35,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:35,696 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159675667, max timestamp=1733159675684Num uniques 859; 2024-12-02T17:14:35,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:35,753 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T17:14:35,754 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T17:14:35,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T17:14:35,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T17:14:35,759 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T17:14:35,759 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T17:14:35,761 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T17:14:35,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T17:14:35,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159675751, max timestamp=1733159675771Num uniques 1018; 2024-12-02T17:14:35,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T17:14:35,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159675751, max timestamp=1733159675771Num uniques 1018; 2024-12-02T17:14:35,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T17:14:35,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159675751, max timestamp=1733159675771Num uniques 1018; 2024-12-02T17:14:35,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-02T17:14:35,870 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/de7fde781b784f239f071c49b505457c 2024-12-02T17:14:35,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733159675824, max timestamp=1733159675858Num uniques 1027; 2024-12-02T17:14:35,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T17:14:35,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733159675824, max timestamp=1733159675858Num uniques 1027; 2024-12-02T17:14:35,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T17:14:35,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733159675824, max timestamp=1733159675858Num uniques 1027; 2024-12-02T17:14:35,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T17:14:35,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/0eb433f6bd9145b886643eb3f34a8f88 is 50, key is test_row_0/C:col10/1733159674385/Put/seqid=0 2024-12-02T17:14:35,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741857_1033 (size=9657) 2024-12-02T17:14:35,954 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/0eb433f6bd9145b886643eb3f34a8f88 2024-12-02T17:14:35,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733159675921, max timestamp=1733159675946Num uniques 1186; 2024-12-02T17:14:35,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T17:14:35,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733159675921, max timestamp=1733159675959Num uniques 1186; 2024-12-02T17:14:35,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T17:14:35,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/e37d94306ca3487eb0f966e607e3628f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/e37d94306ca3487eb0f966e607e3628f 2024-12-02T17:14:35,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8511, totalHeapSize=19584, min timestamp=1733159675921, max timestamp=1733159675959Num uniques 1186; 2024-12-02T17:14:35,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54311, new segment data size=8511, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T17:14:36,009 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/e37d94306ca3487eb0f966e607e3628f, entries=100, sequenceid=132, filesize=9.4 K 2024-12-02T17:14:36,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/de7fde781b784f239f071c49b505457c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/de7fde781b784f239f071c49b505457c 2024-12-02T17:14:36,030 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/de7fde781b784f239f071c49b505457c, entries=100, sequenceid=132, filesize=9.4 K 2024-12-02T17:14:36,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/0eb433f6bd9145b886643eb3f34a8f88 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/0eb433f6bd9145b886643eb3f34a8f88 2024-12-02T17:14:36,051 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/0eb433f6bd9145b886643eb3f34a8f88, entries=100, sequenceid=132, filesize=9.4 K 2024-12-02T17:14:36,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=100.63 KB/103050 for dcecfb16214ccdbe1c27643e3ed0b42c in 761ms, sequenceid=132, compaction requested=true 2024-12-02T17:14:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:36,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T17:14:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T17:14:36,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=107.34 KB heapSize=302.77 KB 2024-12-02T17:14:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:36,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:36,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:36,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T17:14:36,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0810 sec 2024-12-02T17:14:36,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0980 sec 2024-12-02T17:14:36,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/49c9103387544b1ea06730caa72a02c8 is 50, key is test_row_0/A:col10/1733159675921/Put/seqid=0 2024-12-02T17:14:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T17:14:36,108 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:36,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-02T17:14:36,115 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T17:14:36,121 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:36,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:36,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159736116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741858_1034 (size=16931) 2024-12-02T17:14:36,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/49c9103387544b1ea06730caa72a02c8 2024-12-02T17:14:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159736131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159736142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159736149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159736164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/46be562888b842ccbdd7441eb70ae409 is 50, key is test_row_0/B:col10/1733159675921/Put/seqid=0 2024-12-02T17:14:36,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741859_1035 (size=12151) 2024-12-02T17:14:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T17:14:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T17:14:36,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159736274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159736258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159736274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159736284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159736293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T17:14:36,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T17:14:36,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159736488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159736488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159736497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159736507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159736513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T17:14:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:36,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/46be562888b842ccbdd7441eb70ae409 2024-12-02T17:14:36,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/24a6b1ca93b84c069265edb762fdfeec is 50, key is test_row_0/C:col10/1733159675921/Put/seqid=0 2024-12-02T17:14:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T17:14:36,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T17:14:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:36,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:36,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741860_1036 (size=12151) 2024-12-02T17:14:36,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/24a6b1ca93b84c069265edb762fdfeec 2024-12-02T17:14:36,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159736817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159736817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159736818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/49c9103387544b1ea06730caa72a02c8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/49c9103387544b1ea06730caa72a02c8 2024-12-02T17:14:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159736818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159736827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:36,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:36,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/49c9103387544b1ea06730caa72a02c8, entries=250, sequenceid=217, filesize=16.5 K 2024-12-02T17:14:36,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/46be562888b842ccbdd7441eb70ae409 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/46be562888b842ccbdd7441eb70ae409 2024-12-02T17:14:36,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/46be562888b842ccbdd7441eb70ae409, entries=150, sequenceid=217, filesize=11.9 K 2024-12-02T17:14:36,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/24a6b1ca93b84c069265edb762fdfeec as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/24a6b1ca93b84c069265edb762fdfeec 2024-12-02T17:14:36,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/24a6b1ca93b84c069265edb762fdfeec, entries=150, sequenceid=217, filesize=11.9 K 2024-12-02T17:14:36,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~322.64 KB/330384, currentSize=73.80 KB/75570 for dcecfb16214ccdbe1c27643e3ed0b42c in 854ms, sequenceid=217, compaction requested=true 2024-12-02T17:14:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:36,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:36,916 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:36,916 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:36,927 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:36,927 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:36,927 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:36,927 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:36,927 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,928 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,928 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/35c373df4701459cbc9cd775916a4886, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/14a685c335034110b01d4f9c82a5dd3e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/e37d94306ca3487eb0f966e607e3628f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/49c9103387544b1ea06730caa72a02c8] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=49.5 K 2024-12-02T17:14:36,928 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/ff0bf45030fb4cf8b3a523c1f991dcad, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/69f13fd2752847bfad9e80c04d775960, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/de7fde781b784f239f071c49b505457c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/46be562888b842ccbdd7441eb70ae409] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=44.9 K 2024-12-02T17:14:36,929 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0bf45030fb4cf8b3a523c1f991dcad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159672868 2024-12-02T17:14:36,930 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f13fd2752847bfad9e80c04d775960, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733159673685 2024-12-02T17:14:36,930 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c373df4701459cbc9cd775916a4886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159672868 2024-12-02T17:14:36,937 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting de7fde781b784f239f071c49b505457c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733159674383 2024-12-02T17:14:36,938 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a685c335034110b01d4f9c82a5dd3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733159673685 2024-12-02T17:14:36,938 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 46be562888b842ccbdd7441eb70ae409, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733159675921 2024-12-02T17:14:36,938 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e37d94306ca3487eb0f966e607e3628f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733159674383 2024-12-02T17:14:36,939 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49c9103387544b1ea06730caa72a02c8, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733159675921 2024-12-02T17:14:36,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T17:14:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:36,950 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:14:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/4faef52d83034ead9e5f4053f3a965a0 is 50, key is test_row_0/A:col10/1733159676111/Put/seqid=0 2024-12-02T17:14:36,996 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:36,997 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/731ce4f75b664f6cb51289958f3105d2 is 50, key is test_row_0/B:col10/1733159675921/Put/seqid=0 2024-12-02T17:14:37,014 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:37,015 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b137837d41964becadbdc5b0923e8eef is 50, key is test_row_0/A:col10/1733159675921/Put/seqid=0 2024-12-02T17:14:37,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741861_1037 (size=12151) 2024-12-02T17:14:37,084 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/4faef52d83034ead9e5f4053f3a965a0 2024-12-02T17:14:37,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741862_1038 (size=12425) 2024-12-02T17:14:37,128 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/731ce4f75b664f6cb51289958f3105d2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/731ce4f75b664f6cb51289958f3105d2 2024-12-02T17:14:37,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741863_1039 (size=12425) 2024-12-02T17:14:37,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d65ee3866107402eb47a9936ff684e19 is 50, key is test_row_0/B:col10/1733159676111/Put/seqid=0 2024-12-02T17:14:37,145 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b137837d41964becadbdc5b0923e8eef as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b137837d41964becadbdc5b0923e8eef 2024-12-02T17:14:37,155 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 731ce4f75b664f6cb51289958f3105d2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:37,155 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:37,156 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=12, startTime=1733159676916; duration=0sec 2024-12-02T17:14:37,159 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into b137837d41964becadbdc5b0923e8eef(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:37,159 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:37,159 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=12, startTime=1733159676916; duration=0sec 2024-12-02T17:14:37,159 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:37,159 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:37,159 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:37,162 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:37,162 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:37,165 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:37,165 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:37,166 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:37,166 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fd42dd3ca32462db6ff94c41a3c3b7b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/027765d4429348ee9a8cbd691d29c79e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/0eb433f6bd9145b886643eb3f34a8f88, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/24a6b1ca93b84c069265edb762fdfeec] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=44.9 K 2024-12-02T17:14:37,166 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fd42dd3ca32462db6ff94c41a3c3b7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159672868 2024-12-02T17:14:37,167 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027765d4429348ee9a8cbd691d29c79e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733159673685 2024-12-02T17:14:37,168 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb433f6bd9145b886643eb3f34a8f88, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733159674383 2024-12-02T17:14:37,169 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24a6b1ca93b84c069265edb762fdfeec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733159675921 2024-12-02T17:14:37,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741864_1040 (size=12151) 2024-12-02T17:14:37,212 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d65ee3866107402eb47a9936ff684e19 2024-12-02T17:14:37,227 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:37,228 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/8afcd483a74e4b7ea2bcfc6c3efdd86d is 50, key is test_row_0/C:col10/1733159675921/Put/seqid=0 2024-12-02T17:14:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/c613750a238f4c0b9df127bd7f940942 is 50, key is test_row_0/C:col10/1733159676111/Put/seqid=0 2024-12-02T17:14:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T17:14:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741865_1041 (size=12425) 2024-12-02T17:14:37,315 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/8afcd483a74e4b7ea2bcfc6c3efdd86d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8afcd483a74e4b7ea2bcfc6c3efdd86d 2024-12-02T17:14:37,340 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 8afcd483a74e4b7ea2bcfc6c3efdd86d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:37,340 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:37,340 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=12, startTime=1733159676917; duration=0sec 2024-12-02T17:14:37,340 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:37,340 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:37,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741866_1042 (size=12151) 2024-12-02T17:14:37,350 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/c613750a238f4c0b9df127bd7f940942 2024-12-02T17:14:37,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/4faef52d83034ead9e5f4053f3a965a0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4faef52d83034ead9e5f4053f3a965a0 2024-12-02T17:14:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4faef52d83034ead9e5f4053f3a965a0, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T17:14:37,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d65ee3866107402eb47a9936ff684e19 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d65ee3866107402eb47a9936ff684e19 2024-12-02T17:14:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:37,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:37,386 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d65ee3866107402eb47a9936ff684e19, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T17:14:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/c613750a238f4c0b9df127bd7f940942 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/c613750a238f4c0b9df127bd7f940942 2024-12-02T17:14:37,418 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/c613750a238f4c0b9df127bd7f940942, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T17:14:37,424 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=46.96 KB/48090 for dcecfb16214ccdbe1c27643e3ed0b42c in 474ms, sequenceid=231, compaction requested=false 2024-12-02T17:14:37,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:37,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:37,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T17:14:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T17:14:37,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T17:14:37,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3040 sec 2024-12-02T17:14:37,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:14:37,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:37,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:37,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:37,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:37,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:37,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:37,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3200 sec 2024-12-02T17:14:37,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/cff43c427e3f452f8131c2d4ca73f10c is 50, key is test_row_0/A:col10/1733159677386/Put/seqid=0 2024-12-02T17:14:37,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741867_1043 (size=12151) 2024-12-02T17:14:37,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/cff43c427e3f452f8131c2d4ca73f10c 2024-12-02T17:14:37,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/f38ac2a08af94b50a72380446f98b42a is 50, key is test_row_0/B:col10/1733159677386/Put/seqid=0 2024-12-02T17:14:37,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159677506, max timestamp=1733159677540Num uniques 859; 2024-12-02T17:14:37,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:37,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159677506, max timestamp=1733159677540Num uniques 859; 2024-12-02T17:14:37,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:37,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159677506, max timestamp=1733159677540Num uniques 859; 2024-12-02T17:14:37,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:14:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741868_1044 (size=12151) 2024-12-02T17:14:37,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/f38ac2a08af94b50a72380446f98b42a 2024-12-02T17:14:37,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/b9489e54140a4028af601e9f40f70065 is 50, key is test_row_0/C:col10/1733159677386/Put/seqid=0 2024-12-02T17:14:37,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159737618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159737630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159737644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741869_1045 (size=12151) 2024-12-02T17:14:37,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159737657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/b9489e54140a4028af601e9f40f70065 2024-12-02T17:14:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159737661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/cff43c427e3f452f8131c2d4ca73f10c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cff43c427e3f452f8131c2d4ca73f10c 2024-12-02T17:14:37,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cff43c427e3f452f8131c2d4ca73f10c, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T17:14:37,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/f38ac2a08af94b50a72380446f98b42a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f38ac2a08af94b50a72380446f98b42a 2024-12-02T17:14:37,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f38ac2a08af94b50a72380446f98b42a, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T17:14:37,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/b9489e54140a4028af601e9f40f70065 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/b9489e54140a4028af601e9f40f70065 2024-12-02T17:14:37,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/b9489e54140a4028af601e9f40f70065, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T17:14:37,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for dcecfb16214ccdbe1c27643e3ed0b42c in 346ms, sequenceid=245, compaction requested=true 2024-12-02T17:14:37,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:37,778 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:37,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:37,780 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:37,780 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:37,780 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:37,780 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b137837d41964becadbdc5b0923e8eef, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4faef52d83034ead9e5f4053f3a965a0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cff43c427e3f452f8131c2d4ca73f10c] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=35.9 K 2024-12-02T17:14:37,781 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b137837d41964becadbdc5b0923e8eef, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733159675921 2024-12-02T17:14:37,781 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4faef52d83034ead9e5f4053f3a965a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159676111 2024-12-02T17:14:37,783 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:37,784 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting cff43c427e3f452f8131c2d4ca73f10c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159677376 2024-12-02T17:14:37,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-02T17:14:37,788 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:37,789 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:37,789 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:37,789 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/731ce4f75b664f6cb51289958f3105d2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d65ee3866107402eb47a9936ff684e19, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f38ac2a08af94b50a72380446f98b42a] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=35.9 K 2024-12-02T17:14:37,791 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731ce4f75b664f6cb51289958f3105d2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733159675921 2024-12-02T17:14:37,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733159677618, max timestamp=1733159677783Num uniques 1018; 2024-12-02T17:14:37,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-02T17:14:37,793 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d65ee3866107402eb47a9936ff684e19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159676111 2024-12-02T17:14:37,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:37,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:37,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:37,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:14:37,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:37,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:14:37,795 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38ac2a08af94b50a72380446f98b42a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159677376 2024-12-02T17:14:37,812 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:37,815 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/1aabda36f8cb4b8f8e9a8d98e012b41c is 50, key is test_row_0/A:col10/1733159677386/Put/seqid=0 2024-12-02T17:14:37,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b59f7160d48442fe8da68c9716f9d71a is 50, key is test_row_0/A:col10/1733159677782/Put/seqid=0 2024-12-02T17:14:37,850 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:37,851 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/e92e4387ed644052a39b3879af771b7b is 50, key is test_row_0/B:col10/1733159677386/Put/seqid=0 2024-12-02T17:14:37,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159737863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159737863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741871_1047 (size=17181) 2024-12-02T17:14:37,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b59f7160d48442fe8da68c9716f9d71a 2024-12-02T17:14:37,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159737882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159737887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159737887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:37,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:37,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:37,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c0175f97f86a488e936e61c85f1bbaf9 is 50, key is test_row_0/B:col10/1733159677782/Put/seqid=0 2024-12-02T17:14:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741870_1046 (size=12527) 2024-12-02T17:14:37,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741872_1048 (size=12527) 2024-12-02T17:14:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159738003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741873_1049 (size=12301) 2024-12-02T17:14:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159737999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159738026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159738026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c0175f97f86a488e936e61c85f1bbaf9 2024-12-02T17:14:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159738044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,065 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/e92e4387ed644052a39b3879af771b7b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e92e4387ed644052a39b3879af771b7b 2024-12-02T17:14:38,079 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into e92e4387ed644052a39b3879af771b7b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:38,079 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:38,079 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159677778; duration=0sec 2024-12-02T17:14:38,080 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:38,080 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:38,080 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:38,082 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:38,082 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:38,082 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,082 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8afcd483a74e4b7ea2bcfc6c3efdd86d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/c613750a238f4c0b9df127bd7f940942, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/b9489e54140a4028af601e9f40f70065] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=35.9 K 2024-12-02T17:14:38,083 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afcd483a74e4b7ea2bcfc6c3efdd86d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733159675921 2024-12-02T17:14:38,083 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c613750a238f4c0b9df127bd7f940942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159676111 2024-12-02T17:14:38,084 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9489e54140a4028af601e9f40f70065, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159677376 2024-12-02T17:14:38,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/cd1ff723215a4b4c8218d3ea061e96c5 is 50, key is test_row_0/C:col10/1733159677782/Put/seqid=0 2024-12-02T17:14:38,135 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#38 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:38,136 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e61e968db60647cd853c5bcd76a5d6eb is 50, key is test_row_0/C:col10/1733159677386/Put/seqid=0 2024-12-02T17:14:38,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741874_1050 (size=12301) 2024-12-02T17:14:38,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48.80 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/cd1ff723215a4b4c8218d3ea061e96c5 2024-12-02T17:14:38,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b59f7160d48442fe8da68c9716f9d71a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b59f7160d48442fe8da68c9716f9d71a 2024-12-02T17:14:38,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b59f7160d48442fe8da68c9716f9d71a, entries=250, sequenceid=284, filesize=16.8 K 2024-12-02T17:14:38,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c0175f97f86a488e936e61c85f1bbaf9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c0175f97f86a488e936e61c85f1bbaf9 2024-12-02T17:14:38,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741875_1051 (size=12527) 2024-12-02T17:14:38,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c0175f97f86a488e936e61c85f1bbaf9, entries=150, sequenceid=284, filesize=12.0 K 2024-12-02T17:14:38,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/cd1ff723215a4b4c8218d3ea061e96c5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/cd1ff723215a4b4c8218d3ea061e96c5 2024-12-02T17:14:38,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/cd1ff723215a4b4c8218d3ea061e96c5, entries=150, sequenceid=284, filesize=12.0 K 2024-12-02T17:14:38,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.42 KB/114095, heapSize ~316.24 KB/323832, currentSize=73.80 KB/75570 for dcecfb16214ccdbe1c27643e3ed0b42c in 411ms, sequenceid=284, compaction requested=false 2024-12-02T17:14:38,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:38,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:14:38,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:38,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:38,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:38,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T17:14:38,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:38,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-02T17:14:38,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:38,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/7329024ddf574ed58b8e29f406400b75 is 50, key is test_row_0/A:col10/1733159678255/Put/seqid=0 2024-12-02T17:14:38,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:38,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T17:14:38,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:38,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741876_1052 (size=14741) 2024-12-02T17:14:38,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/7329024ddf574ed58b8e29f406400b75 2024-12-02T17:14:38,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/47cf69df0ba04f57aee1fb49b8823371 is 50, key is test_row_0/B:col10/1733159678255/Put/seqid=0 2024-12-02T17:14:38,385 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/1aabda36f8cb4b8f8e9a8d98e012b41c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/1aabda36f8cb4b8f8e9a8d98e012b41c 2024-12-02T17:14:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T17:14:38,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741877_1053 (size=12301) 2024-12-02T17:14:38,404 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 1aabda36f8cb4b8f8e9a8d98e012b41c(size=12.2 K), total size for store is 29.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:38,404 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:38,404 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159677777; duration=0sec 2024-12-02T17:14:38,404 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:38,404 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159738397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159738401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159738405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159738410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159738410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159738528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159738527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159738537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159738537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159738537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,590 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e61e968db60647cd853c5bcd76a5d6eb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e61e968db60647cd853c5bcd76a5d6eb 2024-12-02T17:14:38,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T17:14:38,602 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into e61e968db60647cd853c5bcd76a5d6eb(size=12.2 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:38,602 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:38,602 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159677778; duration=0sec 2024-12-02T17:14:38,602 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:38,602 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:38,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159738747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159738747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159738760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159738761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159738761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:38,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:38,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/47cf69df0ba04f57aee1fb49b8823371 2024-12-02T17:14:38,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1c69998a8c14411b8347b0ccc38c4c65 is 50, key is test_row_0/C:col10/1733159678255/Put/seqid=0 2024-12-02T17:14:38,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741878_1054 (size=12301) 2024-12-02T17:14:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T17:14:38,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:38,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:38,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159739062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159739067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159739077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159739088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159739088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:39,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1c69998a8c14411b8347b0ccc38c4c65 2024-12-02T17:14:39,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/7329024ddf574ed58b8e29f406400b75 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7329024ddf574ed58b8e29f406400b75 2024-12-02T17:14:39,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7329024ddf574ed58b8e29f406400b75, entries=200, sequenceid=300, filesize=14.4 K 2024-12-02T17:14:39,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/47cf69df0ba04f57aee1fb49b8823371 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/47cf69df0ba04f57aee1fb49b8823371 2024-12-02T17:14:39,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/47cf69df0ba04f57aee1fb49b8823371, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T17:14:39,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1c69998a8c14411b8347b0ccc38c4c65 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1c69998a8c14411b8347b0ccc38c4c65 2024-12-02T17:14:39,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1c69998a8c14411b8347b0ccc38c4c65, entries=150, sequenceid=300, filesize=12.0 K 2024-12-02T17:14:39,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for dcecfb16214ccdbe1c27643e3ed0b42c in 1089ms, sequenceid=300, compaction requested=true 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:39,349 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:39,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:39,349 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:39,354 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:39,354 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:39,354 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,354 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/1aabda36f8cb4b8f8e9a8d98e012b41c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b59f7160d48442fe8da68c9716f9d71a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7329024ddf574ed58b8e29f406400b75] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=43.4 K 2024-12-02T17:14:39,354 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:39,355 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:39,355 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,355 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aabda36f8cb4b8f8e9a8d98e012b41c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159677376 2024-12-02T17:14:39,355 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e92e4387ed644052a39b3879af771b7b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c0175f97f86a488e936e61c85f1bbaf9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/47cf69df0ba04f57aee1fb49b8823371] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.3 K 2024-12-02T17:14:39,355 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59f7160d48442fe8da68c9716f9d71a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733159677643 2024-12-02T17:14:39,355 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e92e4387ed644052a39b3879af771b7b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159677376 2024-12-02T17:14:39,363 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7329024ddf574ed58b8e29f406400b75, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733159677832 2024-12-02T17:14:39,363 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c0175f97f86a488e936e61c85f1bbaf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733159677782 2024-12-02T17:14:39,364 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 47cf69df0ba04f57aee1fb49b8823371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733159677832 2024-12-02T17:14:39,387 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:39,388 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/55feb41fbd4941ccbef1a8668aea4c35 is 50, key is test_row_0/B:col10/1733159678255/Put/seqid=0 2024-12-02T17:14:39,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T17:14:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:39,398 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:39,399 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/f6df152254c04b328072c39e03bb8a99 is 50, key is test_row_0/A:col10/1733159678255/Put/seqid=0 2024-12-02T17:14:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T17:14:39,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ca6bfa0e867b49daa32ebb77fd173de2 is 50, key is test_row_0/A:col10/1733159678402/Put/seqid=0 2024-12-02T17:14:39,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741879_1055 (size=12779) 2024-12-02T17:14:39,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741880_1056 (size=12779) 2024-12-02T17:14:39,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741881_1057 (size=12301) 2024-12-02T17:14:39,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ca6bfa0e867b49daa32ebb77fd173de2 2024-12-02T17:14:39,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/05a1915fae534465a5131fa336e28093 is 50, key is test_row_0/B:col10/1733159678402/Put/seqid=0 2024-12-02T17:14:39,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741882_1058 (size=12301) 2024-12-02T17:14:39,536 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/05a1915fae534465a5131fa336e28093 2024-12-02T17:14:39,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/847cb90e679a4f6b9ce3bd85621d3acf is 50, key is test_row_0/C:col10/1733159678402/Put/seqid=0 2024-12-02T17:14:39,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159739612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159739617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741883_1059 (size=12301) 2024-12-02T17:14:39,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,644 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/847cb90e679a4f6b9ce3bd85621d3acf 2024-12-02T17:14:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159739622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159739625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ca6bfa0e867b49daa32ebb77fd173de2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ca6bfa0e867b49daa32ebb77fd173de2 2024-12-02T17:14:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159739634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,666 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ca6bfa0e867b49daa32ebb77fd173de2, entries=150, sequenceid=320, filesize=12.0 K 2024-12-02T17:14:39,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/05a1915fae534465a5131fa336e28093 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/05a1915fae534465a5131fa336e28093 2024-12-02T17:14:39,685 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/05a1915fae534465a5131fa336e28093, entries=150, sequenceid=320, filesize=12.0 K 2024-12-02T17:14:39,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/847cb90e679a4f6b9ce3bd85621d3acf as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/847cb90e679a4f6b9ce3bd85621d3acf 2024-12-02T17:14:39,699 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/847cb90e679a4f6b9ce3bd85621d3acf, entries=150, sequenceid=320, filesize=12.0 K 2024-12-02T17:14:39,701 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for dcecfb16214ccdbe1c27643e3ed0b42c in 309ms, sequenceid=320, compaction requested=true 2024-12-02T17:14:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T17:14:39,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T17:14:39,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T17:14:39,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-12-02T17:14:39,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.4500 sec 2024-12-02T17:14:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:39,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:14:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:39,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/d2e0a2735fd34348aaaaf64e3409d311 is 50, key is test_row_0/A:col10/1733159679601/Put/seqid=0 2024-12-02T17:14:39,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741884_1060 (size=14741) 2024-12-02T17:14:39,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/d2e0a2735fd34348aaaaf64e3409d311 2024-12-02T17:14:39,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159739805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159739807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159739809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159739821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159739823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/33acd482ed3f4915afd878c4e2ce78b5 is 50, key is test_row_0/B:col10/1733159679601/Put/seqid=0 2024-12-02T17:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741885_1061 (size=12301) 2024-12-02T17:14:39,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/33acd482ed3f4915afd878c4e2ce78b5 2024-12-02T17:14:39,866 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/55feb41fbd4941ccbef1a8668aea4c35 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/55feb41fbd4941ccbef1a8668aea4c35 2024-12-02T17:14:39,872 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/f6df152254c04b328072c39e03bb8a99 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f6df152254c04b328072c39e03bb8a99 2024-12-02T17:14:39,881 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 55feb41fbd4941ccbef1a8668aea4c35(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:39,881 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:39,882 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159679349; duration=0sec 2024-12-02T17:14:39,882 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:39,882 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:39,882 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:39,884 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:39,884 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:39,884 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,884 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e61e968db60647cd853c5bcd76a5d6eb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/cd1ff723215a4b4c8218d3ea061e96c5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1c69998a8c14411b8347b0ccc38c4c65, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/847cb90e679a4f6b9ce3bd85621d3acf] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=48.3 K 2024-12-02T17:14:39,885 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e61e968db60647cd853c5bcd76a5d6eb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159677376 2024-12-02T17:14:39,885 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1ff723215a4b4c8218d3ea061e96c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1733159677782 2024-12-02T17:14:39,885 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into f6df152254c04b328072c39e03bb8a99(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:39,886 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:39,886 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159679349; duration=0sec 2024-12-02T17:14:39,886 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:39,886 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c69998a8c14411b8347b0ccc38c4c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733159677832 2024-12-02T17:14:39,886 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:39,886 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 847cb90e679a4f6b9ce3bd85621d3acf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733159678375 2024-12-02T17:14:39,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/9f3107d95ec9490291605bd4e67204dd is 50, key is test_row_0/C:col10/1733159679601/Put/seqid=0 2024-12-02T17:14:39,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741886_1062 (size=12301) 2024-12-02T17:14:39,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159739927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159739927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,935 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:39,936 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/65d0cd7010e34fde80e66713ecbc9def is 50, key is test_row_0/C:col10/1733159678402/Put/seqid=0 2024-12-02T17:14:39,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/9f3107d95ec9490291605bd4e67204dd 2024-12-02T17:14:39,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159739938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159739938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741887_1063 (size=12813) 2024-12-02T17:14:39,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159739938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:39,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/d2e0a2735fd34348aaaaf64e3409d311 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d2e0a2735fd34348aaaaf64e3409d311 2024-12-02T17:14:39,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d2e0a2735fd34348aaaaf64e3409d311, entries=200, sequenceid=335, filesize=14.4 K 2024-12-02T17:14:39,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/33acd482ed3f4915afd878c4e2ce78b5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33acd482ed3f4915afd878c4e2ce78b5 2024-12-02T17:14:39,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33acd482ed3f4915afd878c4e2ce78b5, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T17:14:39,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/9f3107d95ec9490291605bd4e67204dd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9f3107d95ec9490291605bd4e67204dd 2024-12-02T17:14:39,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9f3107d95ec9490291605bd4e67204dd, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T17:14:39,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for dcecfb16214ccdbe1c27643e3ed0b42c in 242ms, sequenceid=335, compaction requested=true 2024-12-02T17:14:39,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:39,988 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:39,991 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:39,991 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:39,991 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:39,991 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f6df152254c04b328072c39e03bb8a99, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ca6bfa0e867b49daa32ebb77fd173de2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d2e0a2735fd34348aaaaf64e3409d311] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=38.9 K 2024-12-02T17:14:39,992 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6df152254c04b328072c39e03bb8a99, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733159677832 2024-12-02T17:14:39,992 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6bfa0e867b49daa32ebb77fd173de2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733159678375 2024-12-02T17:14:39,994 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2e0a2735fd34348aaaaf64e3409d311, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159679601 2024-12-02T17:14:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:40,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:40,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:40,017 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:40,018 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/3c4249cd8a50488ba73d2fab53c46283 is 50, key is test_row_0/A:col10/1733159679601/Put/seqid=0 2024-12-02T17:14:40,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741888_1064 (size=12881) 2024-12-02T17:14:40,081 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/3c4249cd8a50488ba73d2fab53c46283 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/3c4249cd8a50488ba73d2fab53c46283 2024-12-02T17:14:40,102 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 3c4249cd8a50488ba73d2fab53c46283(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:40,102 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:40,102 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159679988; duration=0sec 2024-12-02T17:14:40,103 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:40,103 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:40,103 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-12-02T17:14:40,105 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:14:40,105 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:14:40,105 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. because compaction request was cancelled 2024-12-02T17:14:40,105 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:40,105 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:40,121 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:40,121 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:40,121 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,121 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/55feb41fbd4941ccbef1a8668aea4c35, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/05a1915fae534465a5131fa336e28093, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33acd482ed3f4915afd878c4e2ce78b5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.5 K 2024-12-02T17:14:40,123 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55feb41fbd4941ccbef1a8668aea4c35, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733159677832 2024-12-02T17:14:40,125 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a1915fae534465a5131fa336e28093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733159678375 2024-12-02T17:14:40,129 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33acd482ed3f4915afd878c4e2ce78b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159679601 2024-12-02T17:14:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:40,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:14:40,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159679791, max timestamp=1733159680174Num uniques 859; 2024-12-02T17:14:40,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:14:40,180 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:40,181 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c6c586ac31044026b19cc94495dda540 is 50, key is test_row_0/B:col10/1733159679601/Put/seqid=0 2024-12-02T17:14:40,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159679791, max timestamp=1733159680174Num uniques 859; 2024-12-02T17:14:40,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:14:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:40,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b8d0e52b5d524619a19b39ee34ea0e94 is 50, key is test_row_0/A:col10/1733159680174/Put/seqid=0 2024-12-02T17:14:40,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741889_1065 (size=12881) 2024-12-02T17:14:40,232 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c6c586ac31044026b19cc94495dda540 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c6c586ac31044026b19cc94495dda540 2024-12-02T17:14:40,242 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into c6c586ac31044026b19cc94495dda540(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:40,242 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:40,242 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159680007; duration=0sec 2024-12-02T17:14:40,242 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:40,242 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:40,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741890_1066 (size=22065) 2024-12-02T17:14:40,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b8d0e52b5d524619a19b39ee34ea0e94 2024-12-02T17:14:40,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/79771cb38710482881ccea0bc5898fa9 is 50, key is test_row_0/B:col10/1733159680174/Put/seqid=0 2024-12-02T17:14:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159740264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159680248, max timestamp=1733159680276Num uniques 859; 2024-12-02T17:14:40,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:40,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159680248, max timestamp=1733159680276Num uniques 859; 2024-12-02T17:14:40,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:14:40,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159680248, max timestamp=1733159680276Num uniques 859; 2024-12-02T17:14:40,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:14:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741891_1067 (size=12301) 2024-12-02T17:14:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159740335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159740348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,376 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/65d0cd7010e34fde80e66713ecbc9def as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/65d0cd7010e34fde80e66713ecbc9def 2024-12-02T17:14:40,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159740356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159740356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,397 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 65d0cd7010e34fde80e66713ecbc9def(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:40,397 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:40,397 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=12, startTime=1733159679349; duration=0sec 2024-12-02T17:14:40,397 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:40,397 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:40,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159740397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T17:14:40,428 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:40,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-02T17:14:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T17:14:40,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:40,433 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:40,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:40,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159740457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159740478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159740487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159740488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T17:14:40,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T17:14:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159740617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159740672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159740689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159740707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/79771cb38710482881ccea0bc5898fa9 2024-12-02T17:14:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159740709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/97d890c33c87491f9fc4f8279f6ba6cc is 50, key is test_row_0/C:col10/1733159680174/Put/seqid=0 2024-12-02T17:14:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T17:14:40,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T17:14:40,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741892_1068 (size=12301) 2024-12-02T17:14:40,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T17:14:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:40,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159740927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:40,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159740988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159740999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159741017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159741027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T17:14:41,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T17:14:41,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/97d890c33c87491f9fc4f8279f6ba6cc 2024-12-02T17:14:41,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/b8d0e52b5d524619a19b39ee34ea0e94 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b8d0e52b5d524619a19b39ee34ea0e94 2024-12-02T17:14:41,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b8d0e52b5d524619a19b39ee34ea0e94, entries=350, sequenceid=359, filesize=21.5 K 2024-12-02T17:14:41,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/79771cb38710482881ccea0bc5898fa9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/79771cb38710482881ccea0bc5898fa9 2024-12-02T17:14:41,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/79771cb38710482881ccea0bc5898fa9, entries=150, sequenceid=359, filesize=12.0 K 2024-12-02T17:14:41,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/97d890c33c87491f9fc4f8279f6ba6cc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/97d890c33c87491f9fc4f8279f6ba6cc 2024-12-02T17:14:41,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/97d890c33c87491f9fc4f8279f6ba6cc, entries=150, sequenceid=359, filesize=12.0 K 2024-12-02T17:14:41,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~194.59 KB/199264, currentSize=124.44 KB/127430 for dcecfb16214ccdbe1c27643e3ed0b42c in 1027ms, sequenceid=359, compaction requested=true 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:41,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:14:41,203 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:14:41,203 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:41,204 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:14:41,204 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:14:41,205 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. because compaction request was cancelled 2024-12-02T17:14:41,205 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:41,205 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:14:41,206 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:41,206 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:41,206 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,207 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/65d0cd7010e34fde80e66713ecbc9def, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9f3107d95ec9490291605bd4e67204dd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/97d890c33c87491f9fc4f8279f6ba6cc] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.5 K 2024-12-02T17:14:41,207 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:14:41,207 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65d0cd7010e34fde80e66713ecbc9def, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733159678375 2024-12-02T17:14:41,207 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:14:41,207 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. because compaction request was cancelled 2024-12-02T17:14:41,207 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:41,208 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f3107d95ec9490291605bd4e67204dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159679601 2024-12-02T17:14:41,208 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97d890c33c87491f9fc4f8279f6ba6cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159680160 2024-12-02T17:14:41,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-02T17:14:41,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,229 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=124.44 KB heapSize=353.47 KB 2024-12-02T17:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:41,230 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:41,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:41,231 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/8811144d9cc046f88f304601b58c53bf is 50, key is test_row_0/C:col10/1733159680174/Put/seqid=0 2024-12-02T17:14:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ae5d90a894e0436e88d5421758101c5b is 50, key is test_row_0/A:col10/1733159680348/Put/seqid=0 2024-12-02T17:14:41,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741893_1069 (size=12915) 2024-12-02T17:14:41,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741894_1070 (size=12301) 2024-12-02T17:14:41,264 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ae5d90a894e0436e88d5421758101c5b 2024-12-02T17:14:41,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/230d3abc565a41e0a80ee0b1ab7b8509 is 50, key is test_row_0/B:col10/1733159680348/Put/seqid=0 2024-12-02T17:14:41,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741895_1071 (size=12301) 2024-12-02T17:14:41,298 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/230d3abc565a41e0a80ee0b1ab7b8509 2024-12-02T17:14:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/9c779e90074c408892a21b2297fc562f is 50, key is test_row_0/C:col10/1733159680348/Put/seqid=0 2024-12-02T17:14:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741896_1072 (size=12301) 2024-12-02T17:14:41,327 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/9c779e90074c408892a21b2297fc562f 2024-12-02T17:14:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ae5d90a894e0436e88d5421758101c5b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae5d90a894e0436e88d5421758101c5b 2024-12-02T17:14:41,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae5d90a894e0436e88d5421758101c5b, entries=150, sequenceid=396, filesize=12.0 K 2024-12-02T17:14:41,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/230d3abc565a41e0a80ee0b1ab7b8509 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/230d3abc565a41e0a80ee0b1ab7b8509 2024-12-02T17:14:41,354 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/230d3abc565a41e0a80ee0b1ab7b8509, entries=150, sequenceid=396, filesize=12.0 K 2024-12-02T17:14:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/9c779e90074c408892a21b2297fc562f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9c779e90074c408892a21b2297fc562f 2024-12-02T17:14:41,363 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9c779e90074c408892a21b2297fc562f, entries=150, sequenceid=396, filesize=12.0 K 2024-12-02T17:14:41,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~124.44 KB/127430, heapSize ~353.42 KB/361904, currentSize=0 B/0 for dcecfb16214ccdbe1c27643e3ed0b42c in 135ms, sequenceid=396, compaction requested=true 2024-12-02T17:14:41,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:41,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T17:14:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T17:14:41,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T17:14:41,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 934 msec 2024-12-02T17:14:41,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 942 msec 2024-12-02T17:14:41,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:41,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:41,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/0b46c3325de64aacb57678f5c274102d is 50, key is test_row_0/A:col10/1733159681457/Put/seqid=0 2024-12-02T17:14:41,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741897_1073 (size=14741) 2024-12-02T17:14:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159741551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159741547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159741552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159681540, max timestamp=1733159681552Num uniques 859; 2024-12-02T17:14:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159681540, max timestamp=1733159681552Num uniques 859; 2024-12-02T17:14:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:41,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159681540, max timestamp=1733159681552Num uniques 859; 2024-12-02T17:14:41,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T17:14:41,567 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:41,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-02T17:14:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T17:14:41,569 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:41,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:41,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159741632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159741638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,668 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/8811144d9cc046f88f304601b58c53bf as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8811144d9cc046f88f304601b58c53bf 2024-12-02T17:14:41,676 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 8811144d9cc046f88f304601b58c53bf(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:41,676 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:41,676 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159681202; duration=0sec 2024-12-02T17:14:41,676 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:41,677 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T17:14:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159741667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159741667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T17:14:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159741747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159741747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T17:14:41,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:41,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T17:14:41,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159741887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159741887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/0b46c3325de64aacb57678f5c274102d 2024-12-02T17:14:41,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/e600f468b9fc455dba93ed17b45bec2d is 50, key is test_row_0/B:col10/1733159681457/Put/seqid=0 2024-12-02T17:14:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159741957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741898_1074 (size=12301) 2024-12-02T17:14:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159741957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:41,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/e600f468b9fc455dba93ed17b45bec2d 2024-12-02T17:14:41,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/2bf42a1192db4d97bf71b953bb8c2205 is 50, key is test_row_0/C:col10/1733159681457/Put/seqid=0 2024-12-02T17:14:42,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741899_1075 (size=12301) 2024-12-02T17:14:42,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/2bf42a1192db4d97bf71b953bb8c2205 2024-12-02T17:14:42,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/0b46c3325de64aacb57678f5c274102d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0b46c3325de64aacb57678f5c274102d 2024-12-02T17:14:42,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0b46c3325de64aacb57678f5c274102d, entries=200, sequenceid=406, filesize=14.4 K 2024-12-02T17:14:42,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/e600f468b9fc455dba93ed17b45bec2d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e600f468b9fc455dba93ed17b45bec2d 2024-12-02T17:14:42,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T17:14:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:42,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:42,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e600f468b9fc455dba93ed17b45bec2d, entries=150, sequenceid=406, filesize=12.0 K 2024-12-02T17:14:42,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/2bf42a1192db4d97bf71b953bb8c2205 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2bf42a1192db4d97bf71b953bb8c2205 2024-12-02T17:14:42,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2bf42a1192db4d97bf71b953bb8c2205, entries=150, sequenceid=406, filesize=12.0 K 2024-12-02T17:14:42,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for dcecfb16214ccdbe1c27643e3ed0b42c in 576ms, sequenceid=406, compaction requested=true 2024-12-02T17:14:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:42,068 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:42,068 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:42,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:42,071 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:42,071 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:42,071 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:42,072 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c6c586ac31044026b19cc94495dda540, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/79771cb38710482881ccea0bc5898fa9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/230d3abc565a41e0a80ee0b1ab7b8509, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e600f468b9fc455dba93ed17b45bec2d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=48.6 K 2024-12-02T17:14:42,072 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 61988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:42,072 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:42,072 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:42,072 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/3c4249cd8a50488ba73d2fab53c46283, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b8d0e52b5d524619a19b39ee34ea0e94, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae5d90a894e0436e88d5421758101c5b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0b46c3325de64aacb57678f5c274102d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=60.5 K 2024-12-02T17:14:42,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c586ac31044026b19cc94495dda540, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159679601 2024-12-02T17:14:42,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 79771cb38710482881ccea0bc5898fa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159680160 2024-12-02T17:14:42,073 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c4249cd8a50488ba73d2fab53c46283, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159679601 2024-12-02T17:14:42,074 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 230d3abc565a41e0a80ee0b1ab7b8509, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733159680309 2024-12-02T17:14:42,074 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d0e52b5d524619a19b39ee34ea0e94, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159679791 2024-12-02T17:14:42,074 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e600f468b9fc455dba93ed17b45bec2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159681451 2024-12-02T17:14:42,075 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5d90a894e0436e88d5421758101c5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733159680309 2024-12-02T17:14:42,079 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b46c3325de64aacb57678f5c274102d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159681451 2024-12-02T17:14:42,123 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:42,125 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/9e791d7e432e48b3b834baffb9cc2518 is 50, key is test_row_0/A:col10/1733159681457/Put/seqid=0 2024-12-02T17:14:42,144 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#64 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:42,145 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/dda3d332094240e197eff4b6938ee730 is 50, key is test_row_0/B:col10/1733159681457/Put/seqid=0 2024-12-02T17:14:42,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741900_1076 (size=13017) 2024-12-02T17:14:42,189 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/9e791d7e432e48b3b834baffb9cc2518 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/9e791d7e432e48b3b834baffb9cc2518 2024-12-02T17:14:42,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-02T17:14:42,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:42,193 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T17:14:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:42,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741901_1077 (size=13017) 2024-12-02T17:14:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T17:14:42,210 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 9e791d7e432e48b3b834baffb9cc2518(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:42,211 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:42,211 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=12, startTime=1733159682068; duration=0sec 2024-12-02T17:14:42,211 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:42,211 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:42,211 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/a6de56ade38a44e899fe278ac0d89a0c is 50, key is test_row_0/A:col10/1733159681611/Put/seqid=0 2024-12-02T17:14:42,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:42,214 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:42,214 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:42,214 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:42,214 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8811144d9cc046f88f304601b58c53bf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9c779e90074c408892a21b2297fc562f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2bf42a1192db4d97bf71b953bb8c2205] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.6 K 2024-12-02T17:14:42,215 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8811144d9cc046f88f304601b58c53bf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159680160 2024-12-02T17:14:42,215 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c779e90074c408892a21b2297fc562f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1733159680309 2024-12-02T17:14:42,216 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bf42a1192db4d97bf71b953bb8c2205, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159681451 2024-12-02T17:14:42,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741902_1078 (size=12301) 2024-12-02T17:14:42,218 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/a6de56ade38a44e899fe278ac0d89a0c 2024-12-02T17:14:42,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5a5980c80dc242ce81546a26385f081d is 50, key is test_row_0/B:col10/1733159681611/Put/seqid=0 2024-12-02T17:14:42,249 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:42,250 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e656bb9d5b0e4ef88516ec79fec7b6ea is 50, key is test_row_0/C:col10/1733159681457/Put/seqid=0 2024-12-02T17:14:42,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159742259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159742259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741903_1079 (size=12301) 2024-12-02T17:14:42,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741904_1080 (size=13017) 2024-12-02T17:14:42,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159742267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159742268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159742377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159742377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159742567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,578 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159742587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159742597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,611 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/dda3d332094240e197eff4b6938ee730 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/dda3d332094240e197eff4b6938ee730 2024-12-02T17:14:42,619 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into dda3d332094240e197eff4b6938ee730(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:42,619 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:42,619 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=12, startTime=1733159682068; duration=0sec 2024-12-02T17:14:42,620 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:42,620 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:42,674 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5a5980c80dc242ce81546a26385f081d 2024-12-02T17:14:42,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/7747f099ef1c40e1bd95a459f8c74251 is 50, key is test_row_0/C:col10/1733159681611/Put/seqid=0 2024-12-02T17:14:42,693 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e656bb9d5b0e4ef88516ec79fec7b6ea as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e656bb9d5b0e4ef88516ec79fec7b6ea 2024-12-02T17:14:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741905_1081 (size=12301) 2024-12-02T17:14:42,704 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into e656bb9d5b0e4ef88516ec79fec7b6ea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:42,704 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:42,704 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159682068; duration=0sec 2024-12-02T17:14:42,704 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:42,704 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T17:14:42,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159742787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159742797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159742897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:42,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159742907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:42,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/7747f099ef1c40e1bd95a459f8c74251 2024-12-02T17:14:43,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/a6de56ade38a44e899fe278ac0d89a0c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a6de56ade38a44e899fe278ac0d89a0c 2024-12-02T17:14:43,120 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a6de56ade38a44e899fe278ac0d89a0c, entries=150, sequenceid=444, filesize=12.0 K 2024-12-02T17:14:43,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5a5980c80dc242ce81546a26385f081d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5a5980c80dc242ce81546a26385f081d 2024-12-02T17:14:43,140 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5a5980c80dc242ce81546a26385f081d, entries=150, sequenceid=444, filesize=12.0 K 2024-12-02T17:14:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/7747f099ef1c40e1bd95a459f8c74251 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7747f099ef1c40e1bd95a459f8c74251 2024-12-02T17:14:43,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7747f099ef1c40e1bd95a459f8c74251, entries=150, sequenceid=444, filesize=12.0 K 2024-12-02T17:14:43,156 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for dcecfb16214ccdbe1c27643e3ed0b42c in 963ms, sequenceid=444, compaction requested=false 2024-12-02T17:14:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:43,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T17:14:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T17:14:43,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T17:14:43,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-12-02T17:14:43,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5960 sec 2024-12-02T17:14:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:43,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:43,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:43,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/cbc15b8461d7434a810eb39c7a0e0b1f is 50, key is test_row_0/A:col10/1733159682254/Put/seqid=0 2024-12-02T17:14:43,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741906_1082 (size=14741) 2024-12-02T17:14:43,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159683448, max timestamp=1733159683474Num uniques 859; 2024-12-02T17:14:43,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:43,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159683448, max timestamp=1733159683474Num uniques 859; 2024-12-02T17:14:43,483 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:43,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159683448, max timestamp=1733159683474Num uniques 859; 2024-12-02T17:14:43,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159743531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159743533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159743647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159743647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T17:14:43,728 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:43,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:43,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-02T17:14:43,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:43,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:43,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T17:14:43,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159743807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,815 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159743817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,820 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T17:14:43,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/cbc15b8461d7434a810eb39c7a0e0b1f 2024-12-02T17:14:43,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5f03440ea32f40c38913e1926bdaabe1 is 50, key is test_row_0/B:col10/1733159682254/Put/seqid=0 2024-12-02T17:14:43,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159743857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159743857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:43,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741907_1083 (size=12301) 2024-12-02T17:14:43,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T17:14:43,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:43,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:43,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:43,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5f03440ea32f40c38913e1926bdaabe1 2024-12-02T17:14:43,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:43,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:43,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/a4a9d854dc56441ab5bca49e31f22dcf is 50, key is test_row_0/C:col10/1733159682254/Put/seqid=0 2024-12-02T17:14:43,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741908_1084 (size=12301) 2024-12-02T17:14:43,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/a4a9d854dc56441ab5bca49e31f22dcf 2024-12-02T17:14:43,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/cbc15b8461d7434a810eb39c7a0e0b1f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cbc15b8461d7434a810eb39c7a0e0b1f 2024-12-02T17:14:43,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cbc15b8461d7434a810eb39c7a0e0b1f, entries=200, sequenceid=458, filesize=14.4 K 2024-12-02T17:14:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5f03440ea32f40c38913e1926bdaabe1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5f03440ea32f40c38913e1926bdaabe1 2024-12-02T17:14:43,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5f03440ea32f40c38913e1926bdaabe1, entries=150, sequenceid=458, filesize=12.0 K 2024-12-02T17:14:43,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/a4a9d854dc56441ab5bca49e31f22dcf as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a4a9d854dc56441ab5bca49e31f22dcf 2024-12-02T17:14:43,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a4a9d854dc56441ab5bca49e31f22dcf, entries=150, sequenceid=458, filesize=12.0 K 2024-12-02T17:14:43,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dcecfb16214ccdbe1c27643e3ed0b42c in 557ms, sequenceid=458, compaction requested=true 2024-12-02T17:14:43,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:43,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:43,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:43,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:43,970 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:43,970 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:43,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:43,971 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:43,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:43,971 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:43,971 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/9e791d7e432e48b3b834baffb9cc2518, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a6de56ade38a44e899fe278ac0d89a0c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cbc15b8461d7434a810eb39c7a0e0b1f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=39.1 K 2024-12-02T17:14:43,972 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:43,972 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e791d7e432e48b3b834baffb9cc2518, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159681451 2024-12-02T17:14:43,972 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:43,972 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:43,972 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/dda3d332094240e197eff4b6938ee730, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5a5980c80dc242ce81546a26385f081d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5f03440ea32f40c38913e1926bdaabe1] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.7 K 2024-12-02T17:14:43,972 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6de56ade38a44e899fe278ac0d89a0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733159681611 2024-12-02T17:14:43,974 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting dda3d332094240e197eff4b6938ee730, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159681451 2024-12-02T17:14:43,974 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc15b8461d7434a810eb39c7a0e0b1f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733159682229 2024-12-02T17:14:43,974 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5980c80dc242ce81546a26385f081d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733159681611 2024-12-02T17:14:43,975 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f03440ea32f40c38913e1926bdaabe1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733159682253 2024-12-02T17:14:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:43,991 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:43,992 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/0784a1bbcf9d4400b7322195be734678 is 50, key is test_row_0/B:col10/1733159682254/Put/seqid=0 2024-12-02T17:14:44,000 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:44,002 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/629004400a0443eb8c9b58f9113321b2 is 50, key is test_row_0/A:col10/1733159682254/Put/seqid=0 2024-12-02T17:14:44,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-02T17:14:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,042 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-02T17:14:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:44,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741909_1085 (size=13119) 2024-12-02T17:14:44,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741910_1086 (size=13119) 2024-12-02T17:14:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T17:14:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/8d647d042fe545dc837ccd3d90781e26 is 50, key is test_row_0/A:col10/1733159683522/Put/seqid=0 2024-12-02T17:14:44,056 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/629004400a0443eb8c9b58f9113321b2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/629004400a0443eb8c9b58f9113321b2 2024-12-02T17:14:44,057 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/0784a1bbcf9d4400b7322195be734678 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/0784a1bbcf9d4400b7322195be734678 2024-12-02T17:14:44,066 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 629004400a0443eb8c9b58f9113321b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:44,066 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,066 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159683969; duration=0sec 2024-12-02T17:14:44,066 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:44,066 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:44,066 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:44,067 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:44,067 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:44,067 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,068 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e656bb9d5b0e4ef88516ec79fec7b6ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7747f099ef1c40e1bd95a459f8c74251, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a4a9d854dc56441ab5bca49e31f22dcf] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.7 K 2024-12-02T17:14:44,068 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e656bb9d5b0e4ef88516ec79fec7b6ea, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159681451 2024-12-02T17:14:44,068 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7747f099ef1c40e1bd95a459f8c74251, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733159681611 2024-12-02T17:14:44,069 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a9d854dc56441ab5bca49e31f22dcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733159682253 2024-12-02T17:14:44,073 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 0784a1bbcf9d4400b7322195be734678(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:44,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,073 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159683970; duration=0sec 2024-12-02T17:14:44,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:44,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:44,086 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:44,087 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/2b3d76bc7ba4455cb05d1538dcaca659 is 50, key is test_row_0/C:col10/1733159682254/Put/seqid=0 2024-12-02T17:14:44,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741911_1087 (size=12301) 2024-12-02T17:14:44,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/8d647d042fe545dc837ccd3d90781e26 2024-12-02T17:14:44,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741912_1088 (size=13119) 2024-12-02T17:14:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2d962c1c03d9497ea42411b69a66a34a is 50, key is test_row_0/B:col10/1733159683522/Put/seqid=0 2024-12-02T17:14:44,146 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/2b3d76bc7ba4455cb05d1538dcaca659 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2b3d76bc7ba4455cb05d1538dcaca659 2024-12-02T17:14:44,154 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 2b3d76bc7ba4455cb05d1538dcaca659(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:44,154 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,154 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159683971; duration=0sec 2024-12-02T17:14:44,155 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:44,155 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:44,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741913_1089 (size=12301) 2024-12-02T17:14:44,171 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2d962c1c03d9497ea42411b69a66a34a 2024-12-02T17:14:44,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:44,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/a3ef27e43596452db7393f90e8e6582c is 50, key is test_row_0/C:col10/1733159683522/Put/seqid=0 2024-12-02T17:14:44,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159744205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:44,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159744205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:44,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741914_1090 (size=12301) 2024-12-02T17:14:44,225 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/a3ef27e43596452db7393f90e8e6582c 2024-12-02T17:14:44,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/8d647d042fe545dc837ccd3d90781e26 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d647d042fe545dc837ccd3d90781e26 2024-12-02T17:14:44,240 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d647d042fe545dc837ccd3d90781e26, entries=150, sequenceid=493, filesize=12.0 K 2024-12-02T17:14:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2d962c1c03d9497ea42411b69a66a34a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2d962c1c03d9497ea42411b69a66a34a 2024-12-02T17:14:44,259 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2d962c1c03d9497ea42411b69a66a34a, entries=150, sequenceid=493, filesize=12.0 K 2024-12-02T17:14:44,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/a3ef27e43596452db7393f90e8e6582c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a3ef27e43596452db7393f90e8e6582c 2024-12-02T17:14:44,273 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a3ef27e43596452db7393f90e8e6582c, entries=150, sequenceid=493, filesize=12.0 K 2024-12-02T17:14:44,274 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for dcecfb16214ccdbe1c27643e3ed0b42c in 232ms, sequenceid=493, compaction requested=false 2024-12-02T17:14:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T17:14:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T17:14:44,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T17:14:44,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 544 msec 2024-12-02T17:14:44,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 552 msec 2024-12-02T17:14:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:44,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:14:44,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:44,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T17:14:44,358 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:44,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-02T17:14:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T17:14:44,362 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:44,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:44,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:44,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ff6f087042f74034ab88da9c50a43024 is 50, key is test_row_0/A:col10/1733159684325/Put/seqid=0 2024-12-02T17:14:44,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741915_1091 (size=12301) 2024-12-02T17:14:44,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ff6f087042f74034ab88da9c50a43024 2024-12-02T17:14:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159744422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:44,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/f18c4f5449b14f3889fb65fc3cbf2bb1 is 50, key is test_row_0/B:col10/1733159684325/Put/seqid=0 2024-12-02T17:14:44,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159744424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:44,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741916_1092 (size=12301) 2024-12-02T17:14:44,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/f18c4f5449b14f3889fb65fc3cbf2bb1 2024-12-02T17:14:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T17:14:44,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/271ffe0ca7fd456d8da183c6a65beef4 is 50, key is test_row_0/C:col10/1733159684325/Put/seqid=0 2024-12-02T17:14:44,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741917_1093 (size=12301) 2024-12-02T17:14:44,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/271ffe0ca7fd456d8da183c6a65beef4 2024-12-02T17:14:44,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ff6f087042f74034ab88da9c50a43024 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ff6f087042f74034ab88da9c50a43024 2024-12-02T17:14:44,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ff6f087042f74034ab88da9c50a43024, entries=150, sequenceid=510, filesize=12.0 K 2024-12-02T17:14:44,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/f18c4f5449b14f3889fb65fc3cbf2bb1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f18c4f5449b14f3889fb65fc3cbf2bb1 2024-12-02T17:14:44,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T17:14:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f18c4f5449b14f3889fb65fc3cbf2bb1, entries=150, sequenceid=510, filesize=12.0 K 2024-12-02T17:14:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/271ffe0ca7fd456d8da183c6a65beef4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/271ffe0ca7fd456d8da183c6a65beef4 2024-12-02T17:14:44,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159744537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:44,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/271ffe0ca7fd456d8da183c6a65beef4, entries=150, sequenceid=510, filesize=12.0 K 2024-12-02T17:14:44,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for dcecfb16214ccdbe1c27643e3ed0b42c in 190ms, sequenceid=510, compaction requested=true 2024-12-02T17:14:44,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:44,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:44,546 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:44,546 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:44,549 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:44,549 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:44,549 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:44,549 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:44,549 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,549 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,550 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/0784a1bbcf9d4400b7322195be734678, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2d962c1c03d9497ea42411b69a66a34a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f18c4f5449b14f3889fb65fc3cbf2bb1] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.8 K 2024-12-02T17:14:44,550 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/629004400a0443eb8c9b58f9113321b2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d647d042fe545dc837ccd3d90781e26, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ff6f087042f74034ab88da9c50a43024] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.8 K 2024-12-02T17:14:44,550 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0784a1bbcf9d4400b7322195be734678, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733159682253 2024-12-02T17:14:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:44,551 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 629004400a0443eb8c9b58f9113321b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733159682253 2024-12-02T17:14:44,551 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d962c1c03d9497ea42411b69a66a34a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1733159683521 2024-12-02T17:14:44,552 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d647d042fe545dc837ccd3d90781e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1733159683521 2024-12-02T17:14:44,552 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting f18c4f5449b14f3889fb65fc3cbf2bb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733159684202 2024-12-02T17:14:44,553 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff6f087042f74034ab88da9c50a43024, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733159684202 2024-12-02T17:14:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:44,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:44,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-12-02T17:14:44,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159684417, max timestamp=1733159684562Num uniques 859; 2024-12-02T17:14:44,565 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159684417, max timestamp=1733159684562Num uniques 859; 2024-12-02T17:14:44,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:44,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:44,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159684417, max timestamp=1733159684562Num uniques 859; 2024-12-02T17:14:44,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:44,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:44,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:44,578 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:44,578 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/b47043bef4164373a1a8e433f86dd452 is 50, key is test_row_0/B:col10/1733159684325/Put/seqid=0 2024-12-02T17:14:44,584 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:44,585 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/2894bd25d1c240478d18188bbaebf7a8 is 50, key is test_row_0/A:col10/1733159684325/Put/seqid=0 2024-12-02T17:14:44,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/39a7e3b020454c6cb91f904186fba16e is 50, key is test_row_0/A:col10/1733159684562/Put/seqid=0 2024-12-02T17:14:44,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741918_1094 (size=13221) 2024-12-02T17:14:44,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741919_1095 (size=13221) 2024-12-02T17:14:44,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741920_1096 (size=12301) 2024-12-02T17:14:44,657 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/2894bd25d1c240478d18188bbaebf7a8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/2894bd25d1c240478d18188bbaebf7a8 2024-12-02T17:14:44,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159684633, max timestamp=1733159684665Num uniques 859; 2024-12-02T17:14:44,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159684633, max timestamp=1733159684665Num uniques 859; 2024-12-02T17:14:44,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:44,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159684633, max timestamp=1733159684665Num uniques 859; 2024-12-02T17:14:44,672 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:14:44,674 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 2894bd25d1c240478d18188bbaebf7a8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:44,674 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,674 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159684546; duration=0sec 2024-12-02T17:14:44,674 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:44,674 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:44,675 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T17:14:44,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T17:14:44,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,680 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:44,680 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:44,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,680 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,681 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2b3d76bc7ba4455cb05d1538dcaca659, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a3ef27e43596452db7393f90e8e6582c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/271ffe0ca7fd456d8da183c6a65beef4] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=36.8 K 2024-12-02T17:14:44,681 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3d76bc7ba4455cb05d1538dcaca659, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1733159682253 2024-12-02T17:14:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,686 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ef27e43596452db7393f90e8e6582c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1733159683521 2024-12-02T17:14:44,687 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271ffe0ca7fd456d8da183c6a65beef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733159684202 2024-12-02T17:14:44,707 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:44,708 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/ba9b8763d64a4e5f865735880da4e7ed is 50, key is test_row_0/C:col10/1733159684325/Put/seqid=0 2024-12-02T17:14:44,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741921_1097 (size=13221) 2024-12-02T17:14:44,776 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/ba9b8763d64a4e5f865735880da4e7ed as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ba9b8763d64a4e5f865735880da4e7ed 2024-12-02T17:14:44,785 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into ba9b8763d64a4e5f865735880da4e7ed(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:44,785 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:44,785 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159684563; duration=0sec 2024-12-02T17:14:44,787 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:44,787 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:44,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159684746, max timestamp=1733159684784Num uniques 1068; 2024-12-02T17:14:44,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T17:14:44,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159684746, max timestamp=1733159684784Num uniques 1068; 2024-12-02T17:14:44,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733159684746, max timestamp=1733159684784Num uniques 1068; 2024-12-02T17:14:44,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T17:14:44,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-02T17:14:44,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T17:14:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733159684833, max timestamp=1733159684859Num uniques 1027; 2024-12-02T17:14:44,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733159684833, max timestamp=1733159684859Num uniques 1027; 2024-12-02T17:14:44,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T17:14:44,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T17:14:44,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733159684833, max timestamp=1733159684859Num uniques 1027; 2024-12-02T17:14:44,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-02T17:14:44,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733159684892, max timestamp=1733159684929Num uniques 1186; 2024-12-02T17:14:44,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T17:14:44,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733159684892, max timestamp=1733159684931Num uniques 1186; 2024-12-02T17:14:44,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T17:14:44,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1733159684892, max timestamp=1733159684931Num uniques 1186; 2024-12-02T17:14:44,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54306, new segment data size=8506, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-12-02T17:14:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T17:14:44,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T17:14:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:44,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:45,053 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/b47043bef4164373a1a8e433f86dd452 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/b47043bef4164373a1a8e433f86dd452 2024-12-02T17:14:45,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/39a7e3b020454c6cb91f904186fba16e 2024-12-02T17:14:45,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733159684980, max timestamp=1733159685058Num uniques 1045; 2024-12-02T17:14:45,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-12-02T17:14:45,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733159684980, max timestamp=1733159685058Num uniques 1045; 2024-12-02T17:14:45,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-12-02T17:14:45,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1733159684980, max timestamp=1733159685058Num uniques 1045; 2024-12-02T17:14:45,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-12-02T17:14:45,077 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into b47043bef4164373a1a8e433f86dd452(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:45,077 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:45,077 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159684546; duration=0sec 2024-12-02T17:14:45,077 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:45,077 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:45,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d21027167d874c008b66a4c5cf2514e9 is 50, key is test_row_0/B:col10/1733159684562/Put/seqid=0 2024-12-02T17:14:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159745130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159745137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T17:14:45,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:45,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:45,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:45,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:45,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:45,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159745140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741922_1098 (size=12301) 2024-12-02T17:14:45,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d21027167d874c008b66a4c5cf2514e9 2024-12-02T17:14:45,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e7613309e60540edaa5bd4ec1cfa175a is 50, key is test_row_0/C:col10/1733159684562/Put/seqid=0 2024-12-02T17:14:45,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741923_1099 (size=12301) 2024-12-02T17:14:45,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e7613309e60540edaa5bd4ec1cfa175a 2024-12-02T17:14:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159745248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159745248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/39a7e3b020454c6cb91f904186fba16e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39a7e3b020454c6cb91f904186fba16e 2024-12-02T17:14:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39a7e3b020454c6cb91f904186fba16e, entries=150, sequenceid=531, filesize=12.0 K 2024-12-02T17:14:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d21027167d874c008b66a4c5cf2514e9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d21027167d874c008b66a4c5cf2514e9 2024-12-02T17:14:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159745257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d21027167d874c008b66a4c5cf2514e9, entries=150, sequenceid=531, filesize=12.0 K 2024-12-02T17:14:45,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e7613309e60540edaa5bd4ec1cfa175a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e7613309e60540edaa5bd4ec1cfa175a 2024-12-02T17:14:45,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e7613309e60540edaa5bd4ec1cfa175a, entries=150, sequenceid=531, filesize=12.0 K 2024-12-02T17:14:45,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~31.31 KB/32060, heapSize ~76.34 KB/78176, currentSize=161.02 KB/164880 for dcecfb16214ccdbe1c27643e3ed0b42c in 724ms, sequenceid=531, compaction requested=false 2024-12-02T17:14:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:45,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:45,297 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=161.02 KB heapSize=447.23 KB 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:45,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/82943e60d5c34d24b54ee8462eec19d0 is 50, key is test_row_0/A:col10/1733159685135/Put/seqid=0 2024-12-02T17:14:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741924_1100 (size=12301) 2024-12-02T17:14:45,340 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/82943e60d5c34d24b54ee8462eec19d0 2024-12-02T17:14:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5014f63a77ba4445a86b2b9197283dbf is 50, key is test_row_0/B:col10/1733159685135/Put/seqid=0 2024-12-02T17:14:45,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741925_1101 (size=12301) 2024-12-02T17:14:45,359 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5014f63a77ba4445a86b2b9197283dbf 2024-12-02T17:14:45,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/44f5148824bb4dfbaff95937d1ec1b09 is 50, key is test_row_0/C:col10/1733159685135/Put/seqid=0 2024-12-02T17:14:45,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741926_1102 (size=12301) 2024-12-02T17:14:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:45,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159745466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159745473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159745477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T17:14:45,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159745577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159745577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159745787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159745787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159745787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,799 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/44f5148824bb4dfbaff95937d1ec1b09 2024-12-02T17:14:45,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/82943e60d5c34d24b54ee8462eec19d0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/82943e60d5c34d24b54ee8462eec19d0 2024-12-02T17:14:45,826 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/82943e60d5c34d24b54ee8462eec19d0, entries=150, sequenceid=640, filesize=12.0 K 2024-12-02T17:14:45,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/5014f63a77ba4445a86b2b9197283dbf as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5014f63a77ba4445a86b2b9197283dbf 2024-12-02T17:14:45,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159745827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:45,832 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:45,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:45,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:45,835 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5014f63a77ba4445a86b2b9197283dbf, entries=150, sequenceid=640, filesize=12.0 K 2024-12-02T17:14:45,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/44f5148824bb4dfbaff95937d1ec1b09 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/44f5148824bb4dfbaff95937d1ec1b09 2024-12-02T17:14:45,845 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/44f5148824bb4dfbaff95937d1ec1b09, entries=150, sequenceid=640, filesize=12.0 K 2024-12-02T17:14:45,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~447.19 KB/457920, currentSize=26.84 KB/27480 for dcecfb16214ccdbe1c27643e3ed0b42c in 550ms, sequenceid=640, compaction requested=true 2024-12-02T17:14:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-02T17:14:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-02T17:14:45,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-02T17:14:45,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4840 sec 2024-12-02T17:14:45,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4940 sec 2024-12-02T17:14:45,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:14:45,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:45,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:45,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:45,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/bde65ae89df5410497af2f33989850b0 is 50, key is test_row_1/A:col10/1733159685853/Put/seqid=0 2024-12-02T17:14:45,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741927_1103 (size=12297) 2024-12-02T17:14:46,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159685985, max timestamp=1733159686001Num uniques 859; 2024-12-02T17:14:46,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:46,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159685985, max timestamp=1733159686001Num uniques 859; 2024-12-02T17:14:46,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:46,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159685985, max timestamp=1733159686001Num uniques 859; 2024-12-02T17:14:46,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:46,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159746097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159746102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159746106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159746217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159746217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159746297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/bde65ae89df5410497af2f33989850b0 2024-12-02T17:14:46,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d112d041b1af4531a14bcca27de8baf8 is 50, key is test_row_1/B:col10/1733159685853/Put/seqid=0 2024-12-02T17:14:46,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741928_1104 (size=9857) 2024-12-02T17:14:46,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d112d041b1af4531a14bcca27de8baf8 2024-12-02T17:14:46,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/bd12199a88dc4d9ca6a4b189abba0453 is 50, key is test_row_1/C:col10/1733159685853/Put/seqid=0 2024-12-02T17:14:46,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741929_1105 (size=9857) 2024-12-02T17:14:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159746427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159746427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T17:14:46,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:46,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-02T17:14:46,510 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T17:14:46,511 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:46,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T17:14:46,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159746617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T17:14:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:46,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:46,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:46,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159746736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159746747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:46,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/bd12199a88dc4d9ca6a4b189abba0453 2024-12-02T17:14:46,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/bde65ae89df5410497af2f33989850b0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bde65ae89df5410497af2f33989850b0 2024-12-02T17:14:46,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bde65ae89df5410497af2f33989850b0, entries=150, sequenceid=650, filesize=12.0 K 2024-12-02T17:14:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/d112d041b1af4531a14bcca27de8baf8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d112d041b1af4531a14bcca27de8baf8 2024-12-02T17:14:46,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d112d041b1af4531a14bcca27de8baf8, entries=100, sequenceid=650, filesize=9.6 K 2024-12-02T17:14:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/bd12199a88dc4d9ca6a4b189abba0453 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/bd12199a88dc4d9ca6a4b189abba0453 2024-12-02T17:14:46,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T17:14:46,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:46,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:46,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:46,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T17:14:46,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/bd12199a88dc4d9ca6a4b189abba0453, entries=100, sequenceid=650, filesize=9.6 K 2024-12-02T17:14:46,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dcecfb16214ccdbe1c27643e3ed0b42c in 968ms, sequenceid=650, compaction requested=true 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:46,834 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:46,834 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:46,837 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:46,837 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:46,837 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,837 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/b47043bef4164373a1a8e433f86dd452, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d21027167d874c008b66a4c5cf2514e9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5014f63a77ba4445a86b2b9197283dbf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d112d041b1af4531a14bcca27de8baf8] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=46.6 K 2024-12-02T17:14:46,837 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b47043bef4164373a1a8e433f86dd452, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733159684202 2024-12-02T17:14:46,838 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50120 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:46,838 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:46,838 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d21027167d874c008b66a4c5cf2514e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733159684419 2024-12-02T17:14:46,838 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,838 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/2894bd25d1c240478d18188bbaebf7a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39a7e3b020454c6cb91f904186fba16e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/82943e60d5c34d24b54ee8462eec19d0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bde65ae89df5410497af2f33989850b0] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=48.9 K 2024-12-02T17:14:46,838 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5014f63a77ba4445a86b2b9197283dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733159685126 2024-12-02T17:14:46,838 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2894bd25d1c240478d18188bbaebf7a8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733159684202 2024-12-02T17:14:46,840 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a7e3b020454c6cb91f904186fba16e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733159684419 2024-12-02T17:14:46,840 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d112d041b1af4531a14bcca27de8baf8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733159685853 2024-12-02T17:14:46,841 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82943e60d5c34d24b54ee8462eec19d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733159685126 2024-12-02T17:14:46,841 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde65ae89df5410497af2f33989850b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733159685849 2024-12-02T17:14:46,852 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:46,853 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:46,855 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/79d902738e4a4a9183fc0b57b3a26f30 is 50, key is test_row_0/A:col10/1733159685135/Put/seqid=0 2024-12-02T17:14:46,856 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2fdc2ee19eec457f8ab3d064e94bc39f is 50, key is test_row_0/B:col10/1733159685135/Put/seqid=0 2024-12-02T17:14:46,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741930_1106 (size=13357) 2024-12-02T17:14:46,905 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/79d902738e4a4a9183fc0b57b3a26f30 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/79d902738e4a4a9183fc0b57b3a26f30 2024-12-02T17:14:46,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741931_1107 (size=13357) 2024-12-02T17:14:46,914 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 79d902738e4a4a9183fc0b57b3a26f30(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:46,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:46,914 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=12, startTime=1733159686834; duration=0sec 2024-12-02T17:14:46,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:46,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:46,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:14:46,916 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:14:46,916 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:46,916 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,916 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ba9b8763d64a4e5f865735880da4e7ed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e7613309e60540edaa5bd4ec1cfa175a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/44f5148824bb4dfbaff95937d1ec1b09, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/bd12199a88dc4d9ca6a4b189abba0453] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=46.6 K 2024-12-02T17:14:46,916 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba9b8763d64a4e5f865735880da4e7ed, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1733159684202 2024-12-02T17:14:46,917 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2fdc2ee19eec457f8ab3d064e94bc39f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2fdc2ee19eec457f8ab3d064e94bc39f 2024-12-02T17:14:46,918 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7613309e60540edaa5bd4ec1cfa175a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1733159684419 2024-12-02T17:14:46,919 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f5148824bb4dfbaff95937d1ec1b09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=640, earliestPutTs=1733159685126 2024-12-02T17:14:46,920 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd12199a88dc4d9ca6a4b189abba0453, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733159685853 2024-12-02T17:14:46,924 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 2fdc2ee19eec457f8ab3d064e94bc39f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:46,924 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:46,924 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=12, startTime=1733159686834; duration=0sec 2024-12-02T17:14:46,925 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:46,925 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:46,936 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#95 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:46,936 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1fe1d697220340e3808fd6bee7c965bf is 50, key is test_row_0/C:col10/1733159685135/Put/seqid=0 2024-12-02T17:14:46,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-02T17:14:46,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:46,977 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T17:14:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741932_1108 (size=13357) 2024-12-02T17:14:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:46,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:46,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/a18ad75b77fc42679df84bff846a6d5b is 50, key is test_row_0/A:col10/1733159686103/Put/seqid=0 2024-12-02T17:14:46,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741933_1109 (size=12301) 2024-12-02T17:14:46,990 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/a18ad75b77fc42679df84bff846a6d5b 2024-12-02T17:14:47,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/44d19b6a403348ea8b05202ed7eed96f is 50, key is test_row_0/B:col10/1733159686103/Put/seqid=0 2024-12-02T17:14:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741934_1110 (size=12301) 2024-12-02T17:14:47,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/44d19b6a403348ea8b05202ed7eed96f 2024-12-02T17:14:47,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1d24afa4cca54742a01dbb20dc6389d0 is 50, key is test_row_0/C:col10/1733159686103/Put/seqid=0 2024-12-02T17:14:47,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741935_1111 (size=12301) 2024-12-02T17:14:47,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=689 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1d24afa4cca54742a01dbb20dc6389d0 2024-12-02T17:14:47,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/a18ad75b77fc42679df84bff846a6d5b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a18ad75b77fc42679df84bff846a6d5b 2024-12-02T17:14:47,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a18ad75b77fc42679df84bff846a6d5b, entries=150, sequenceid=689, filesize=12.0 K 2024-12-02T17:14:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/44d19b6a403348ea8b05202ed7eed96f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/44d19b6a403348ea8b05202ed7eed96f 2024-12-02T17:14:47,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/44d19b6a403348ea8b05202ed7eed96f, entries=150, sequenceid=689, filesize=12.0 K 2024-12-02T17:14:47,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1d24afa4cca54742a01dbb20dc6389d0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1d24afa4cca54742a01dbb20dc6389d0 2024-12-02T17:14:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T17:14:47,146 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1d24afa4cca54742a01dbb20dc6389d0, entries=150, sequenceid=689, filesize=12.0 K 2024-12-02T17:14:47,151 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for dcecfb16214ccdbe1c27643e3ed0b42c in 174ms, sequenceid=689, compaction requested=false 2024-12-02T17:14:47,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:47,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-02T17:14:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-02T17:14:47,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-02T17:14:47,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 643 msec 2024-12-02T17:14:47,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 651 msec 2024-12-02T17:14:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:47,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:14:47,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:47,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:47,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:47,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:47,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/39afe24997b246bab2d8aa11f7a5f04f is 50, key is test_row_0/A:col10/1733159687282/Put/seqid=0 2024-12-02T17:14:47,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741936_1112 (size=17177) 2024-12-02T17:14:47,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159687297, max timestamp=1733159687342Num uniques 859; 2024-12-02T17:14:47,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:47,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159687297, max timestamp=1733159687342Num uniques 859; 2024-12-02T17:14:47,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:47,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159687297, max timestamp=1733159687342Num uniques 859; 2024-12-02T17:14:47,347 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:14:47,387 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/1fe1d697220340e3808fd6bee7c965bf as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fe1d697220340e3808fd6bee7c965bf 2024-12-02T17:14:47,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159747388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159747391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,398 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 1fe1d697220340e3808fd6bee7c965bf(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:47,398 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:47,399 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=12, startTime=1733159686834; duration=0sec 2024-12-02T17:14:47,399 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:47,399 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:47,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159747394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159747496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159747507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159747507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159747637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-02T17:14:47,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:47,644 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-02T17:14:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T17:14:47,651 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:47,652 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:47,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:47,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159747706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159747717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159747717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:47,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/39afe24997b246bab2d8aa11f7a5f04f 2024-12-02T17:14:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T17:14:47,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/34cb23375b8c4133a3f874e54ca407e5 is 50, key is test_row_0/B:col10/1733159687282/Put/seqid=0 2024-12-02T17:14:47,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741937_1113 (size=9857) 2024-12-02T17:14:47,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/34cb23375b8c4133a3f874e54ca407e5 2024-12-02T17:14:47,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T17:14:47,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/793981de67ba46cd8acf612ff16ea059 is 50, key is test_row_0/C:col10/1733159687282/Put/seqid=0 2024-12-02T17:14:47,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:47,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:47,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741938_1114 (size=9857) 2024-12-02T17:14:47,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/793981de67ba46cd8acf612ff16ea059 2024-12-02T17:14:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/39afe24997b246bab2d8aa11f7a5f04f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39afe24997b246bab2d8aa11f7a5f04f 2024-12-02T17:14:47,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39afe24997b246bab2d8aa11f7a5f04f, entries=250, sequenceid=699, filesize=16.8 K 2024-12-02T17:14:47,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/34cb23375b8c4133a3f874e54ca407e5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/34cb23375b8c4133a3f874e54ca407e5 2024-12-02T17:14:47,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/34cb23375b8c4133a3f874e54ca407e5, entries=100, sequenceid=699, filesize=9.6 K 2024-12-02T17:14:47,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/793981de67ba46cd8acf612ff16ea059 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/793981de67ba46cd8acf612ff16ea059 2024-12-02T17:14:47,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/793981de67ba46cd8acf612ff16ea059, entries=100, sequenceid=699, filesize=9.6 K 2024-12-02T17:14:47,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=143.13 KB/146560 for dcecfb16214ccdbe1c27643e3ed0b42c in 593ms, sequenceid=699, compaction requested=true 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:47,879 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:14:47,881 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:47,881 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:47,881 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,881 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/79d902738e4a4a9183fc0b57b3a26f30, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a18ad75b77fc42679df84bff846a6d5b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39afe24997b246bab2d8aa11f7a5f04f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=41.8 K 2024-12-02T17:14:47,882 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:47,882 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d902738e4a4a9183fc0b57b3a26f30, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733159685135 2024-12-02T17:14:47,883 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18ad75b77fc42679df84bff846a6d5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733159686075 2024-12-02T17:14:47,883 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39afe24997b246bab2d8aa11f7a5f04f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733159687264 2024-12-02T17:14:47,883 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:47,883 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:47,883 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,884 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2fdc2ee19eec457f8ab3d064e94bc39f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/44d19b6a403348ea8b05202ed7eed96f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/34cb23375b8c4133a3f874e54ca407e5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=34.7 K 2024-12-02T17:14:47,884 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fdc2ee19eec457f8ab3d064e94bc39f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733159685135 2024-12-02T17:14:47,885 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d19b6a403348ea8b05202ed7eed96f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733159686075 2024-12-02T17:14:47,888 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 34cb23375b8c4133a3f874e54ca407e5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733159687279 2024-12-02T17:14:47,903 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:47,903 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/163d5218f2bc4b3888d252709e14ee92 is 50, key is test_row_0/A:col10/1733159687282/Put/seqid=0 2024-12-02T17:14:47,906 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#103 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:47,907 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/a4ff7ac8fd8e42edac5fd7805f4d3455 is 50, key is test_row_0/B:col10/1733159687282/Put/seqid=0 2024-12-02T17:14:47,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741939_1115 (size=13459) 2024-12-02T17:14:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741940_1116 (size=13459) 2024-12-02T17:14:47,950 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/163d5218f2bc4b3888d252709e14ee92 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/163d5218f2bc4b3888d252709e14ee92 2024-12-02T17:14:47,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-02T17:14:47,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,964 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=143.13 KB heapSize=405.30 KB 2024-12-02T17:14:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:47,966 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 163d5218f2bc4b3888d252709e14ee92(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:47,966 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:47,966 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159687879; duration=0sec 2024-12-02T17:14:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T17:14:47,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:47,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:47,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:47,971 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:47,971 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:47,971 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:47,971 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fe1d697220340e3808fd6bee7c965bf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1d24afa4cca54742a01dbb20dc6389d0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/793981de67ba46cd8acf612ff16ea059] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=34.7 K 2024-12-02T17:14:47,971 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe1d697220340e3808fd6bee7c965bf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1733159685135 2024-12-02T17:14:47,972 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d24afa4cca54742a01dbb20dc6389d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=689, earliestPutTs=1733159686075 2024-12-02T17:14:47,972 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 793981de67ba46cd8acf612ff16ea059, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733159687279 2024-12-02T17:14:47,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/f107302be3d7409db5b3b1e01d7af595 is 50, key is test_row_0/A:col10/1733159687389/Put/seqid=0 2024-12-02T17:14:48,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741941_1117 (size=12301) 2024-12-02T17:14:48,016 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/f107302be3d7409db5b3b1e01d7af595 2024-12-02T17:14:48,019 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:48,020 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/060c40ec45c941dba22313637aec883e is 50, key is test_row_0/C:col10/1733159687282/Put/seqid=0 2024-12-02T17:14:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:48,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2445be97285e40ea9b4bb7a7d2f1bdf2 is 50, key is test_row_0/B:col10/1733159687389/Put/seqid=0 2024-12-02T17:14:48,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159748048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159748048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159748050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741942_1118 (size=13459) 2024-12-02T17:14:48,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741943_1119 (size=12301) 2024-12-02T17:14:48,075 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2445be97285e40ea9b4bb7a7d2f1bdf2 2024-12-02T17:14:48,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/05b966642cb24817b8d62552d85c17f4 is 50, key is test_row_0/C:col10/1733159687389/Put/seqid=0 2024-12-02T17:14:48,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741944_1120 (size=12301) 2024-12-02T17:14:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159748157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159748157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159748157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T17:14:48,358 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/a4ff7ac8fd8e42edac5fd7805f4d3455 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/a4ff7ac8fd8e42edac5fd7805f4d3455 2024-12-02T17:14:48,379 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into a4ff7ac8fd8e42edac5fd7805f4d3455(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:48,379 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:48,379 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159687879; duration=0sec 2024-12-02T17:14:48,379 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:48,379 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:48,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159748377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159748378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159748377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,476 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/060c40ec45c941dba22313637aec883e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/060c40ec45c941dba22313637aec883e 2024-12-02T17:14:48,484 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 060c40ec45c941dba22313637aec883e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:48,484 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:48,484 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159687879; duration=0sec 2024-12-02T17:14:48,485 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:48,485 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:48,564 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/05b966642cb24817b8d62552d85c17f4 2024-12-02T17:14:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/f107302be3d7409db5b3b1e01d7af595 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f107302be3d7409db5b3b1e01d7af595 2024-12-02T17:14:48,577 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f107302be3d7409db5b3b1e01d7af595, entries=150, sequenceid=738, filesize=12.0 K 2024-12-02T17:14:48,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2445be97285e40ea9b4bb7a7d2f1bdf2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2445be97285e40ea9b4bb7a7d2f1bdf2 2024-12-02T17:14:48,584 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2445be97285e40ea9b4bb7a7d2f1bdf2, entries=150, sequenceid=738, filesize=12.0 K 2024-12-02T17:14:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/05b966642cb24817b8d62552d85c17f4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/05b966642cb24817b8d62552d85c17f4 2024-12-02T17:14:48,593 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/05b966642cb24817b8d62552d85c17f4, entries=150, sequenceid=738, filesize=12.0 K 2024-12-02T17:14:48,594 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~405.25 KB/414976, currentSize=40.25 KB/41220 for dcecfb16214ccdbe1c27643e3ed0b42c in 630ms, sequenceid=738, compaction requested=false 2024-12-02T17:14:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:48,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-02T17:14:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-02T17:14:48,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T17:14:48,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 944 msec 2024-12-02T17:14:48,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 951 msec 2024-12-02T17:14:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:48,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:14:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:48,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:48,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/0d0ff3bc34264dda83f5d9bcb7b6b25f is 50, key is test_row_0/A:col10/1733159688689/Put/seqid=0 2024-12-02T17:14:48,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741945_1121 (size=14741) 2024-12-02T17:14:48,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/0d0ff3bc34264dda83f5d9bcb7b6b25f 2024-12-02T17:14:48,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2cdc9a1ad0d0443880b35d77849d29d5 is 50, key is test_row_0/B:col10/1733159688689/Put/seqid=0 2024-12-02T17:14:48,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159688710, max timestamp=1733159688734Num uniques 859; 2024-12-02T17:14:48,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159688710, max timestamp=1733159688734Num uniques 859; 2024-12-02T17:14:48,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:14:48,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159688710, max timestamp=1733159688734Num uniques 859; 2024-12-02T17:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:14:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741946_1122 (size=12301) 2024-12-02T17:14:48,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2cdc9a1ad0d0443880b35d77849d29d5 2024-12-02T17:14:48,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/96ae971fc7d64f588851afe58ed150d5 is 50, key is test_row_0/C:col10/1733159688689/Put/seqid=0 2024-12-02T17:14:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T17:14:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159748780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,788 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:14:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-02T17:14:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:48,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:14:48,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159748789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:14:48,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:14:48,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159748794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741947_1123 (size=12301) 2024-12-02T17:14:48,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=750 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/96ae971fc7d64f588851afe58ed150d5 2024-12-02T17:14:48,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/0d0ff3bc34264dda83f5d9bcb7b6b25f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0d0ff3bc34264dda83f5d9bcb7b6b25f 2024-12-02T17:14:48,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0d0ff3bc34264dda83f5d9bcb7b6b25f, entries=200, sequenceid=750, filesize=14.4 K 2024-12-02T17:14:48,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2cdc9a1ad0d0443880b35d77849d29d5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2cdc9a1ad0d0443880b35d77849d29d5 2024-12-02T17:14:48,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2cdc9a1ad0d0443880b35d77849d29d5, entries=150, sequenceid=750, filesize=12.0 K 2024-12-02T17:14:48,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/96ae971fc7d64f588851afe58ed150d5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/96ae971fc7d64f588851afe58ed150d5 2024-12-02T17:14:48,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/96ae971fc7d64f588851afe58ed150d5, entries=150, sequenceid=750, filesize=12.0 K 2024-12-02T17:14:48,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for dcecfb16214ccdbe1c27643e3ed0b42c in 172ms, sequenceid=750, compaction requested=true 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:48,862 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:48,862 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:48,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:48,863 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:48,864 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:48,864 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:48,864 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/a4ff7ac8fd8e42edac5fd7805f4d3455, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2445be97285e40ea9b4bb7a7d2f1bdf2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2cdc9a1ad0d0443880b35d77849d29d5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.2 K 2024-12-02T17:14:48,866 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:48,866 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:48,866 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:48,866 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ff7ac8fd8e42edac5fd7805f4d3455, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733159686075 2024-12-02T17:14:48,867 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/163d5218f2bc4b3888d252709e14ee92, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f107302be3d7409db5b3b1e01d7af595, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0d0ff3bc34264dda83f5d9bcb7b6b25f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=39.6 K 2024-12-02T17:14:48,867 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2445be97285e40ea9b4bb7a7d2f1bdf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733159687384 2024-12-02T17:14:48,867 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163d5218f2bc4b3888d252709e14ee92, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733159686075 2024-12-02T17:14:48,867 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cdc9a1ad0d0443880b35d77849d29d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733159688041 2024-12-02T17:14:48,868 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f107302be3d7409db5b3b1e01d7af595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733159687384 2024-12-02T17:14:48,869 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0ff3bc34264dda83f5d9bcb7b6b25f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733159688041 2024-12-02T17:14:48,890 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:48,891 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/35594f8161604d11ac298123142e7f96 is 50, key is test_row_0/B:col10/1733159688689/Put/seqid=0 2024-12-02T17:14:48,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-02T17:14:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:48,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:48,907 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:48,908 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/7dd507c7fbe549299970dd5067928d39 is 50, key is test_row_0/A:col10/1733159688689/Put/seqid=0 2024-12-02T17:14:48,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ef7431ba04404ba78a091239aecf7757 is 50, key is test_row_0/A:col10/1733159688898/Put/seqid=0 2024-12-02T17:14:48,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741948_1124 (size=13561) 2024-12-02T17:14:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159748921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159748921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,935 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/35594f8161604d11ac298123142e7f96 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/35594f8161604d11ac298123142e7f96 2024-12-02T17:14:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159748923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:48,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:48,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:48,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:48,954 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 35594f8161604d11ac298123142e7f96(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:48,954 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:48,954 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159688862; duration=0sec 2024-12-02T17:14:48,954 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:48,954 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:48,954 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:48,956 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:48,956 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:48,956 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:48,956 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/060c40ec45c941dba22313637aec883e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/05b966642cb24817b8d62552d85c17f4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/96ae971fc7d64f588851afe58ed150d5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.2 K 2024-12-02T17:14:48,957 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 060c40ec45c941dba22313637aec883e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=699, earliestPutTs=1733159686075 2024-12-02T17:14:48,957 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 05b966642cb24817b8d62552d85c17f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1733159687384 2024-12-02T17:14:48,958 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ae971fc7d64f588851afe58ed150d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733159688041 2024-12-02T17:14:48,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741949_1125 (size=13561) 2024-12-02T17:14:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741950_1126 (size=14741) 2024-12-02T17:14:48,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ef7431ba04404ba78a091239aecf7757 2024-12-02T17:14:48,993 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/7dd507c7fbe549299970dd5067928d39 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7dd507c7fbe549299970dd5067928d39 2024-12-02T17:14:49,002 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:49,002 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/fc12086a811f487c9046eabe7e331144 is 50, key is test_row_0/C:col10/1733159688689/Put/seqid=0 2024-12-02T17:14:49,004 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 7dd507c7fbe549299970dd5067928d39(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:49,004 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:49,004 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159688862; duration=0sec 2024-12-02T17:14:49,004 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:49,004 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:49,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c808dc00bf4e4c5aa6a1eca1525e1e80 is 50, key is test_row_0/B:col10/1733159688898/Put/seqid=0 2024-12-02T17:14:49,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741951_1127 (size=13561) 2024-12-02T17:14:49,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741952_1128 (size=12301) 2024-12-02T17:14:49,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c808dc00bf4e4c5aa6a1eca1525e1e80 2024-12-02T17:14:49,043 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/fc12086a811f487c9046eabe7e331144 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/fc12086a811f487c9046eabe7e331144 2024-12-02T17:14:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159749037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159749038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,055 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into fc12086a811f487c9046eabe7e331144(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:49,055 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:49,055 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159688862; duration=0sec 2024-12-02T17:14:49,055 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:49,055 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:49,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e38dab56e33b448a85987a79b5b6accb is 50, key is test_row_0/C:col10/1733159688898/Put/seqid=0 2024-12-02T17:14:49,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159749047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741953_1129 (size=12301) 2024-12-02T17:14:49,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=786 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e38dab56e33b448a85987a79b5b6accb 2024-12-02T17:14:49,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/ef7431ba04404ba78a091239aecf7757 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ef7431ba04404ba78a091239aecf7757 2024-12-02T17:14:49,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ef7431ba04404ba78a091239aecf7757, entries=200, sequenceid=786, filesize=14.4 K 2024-12-02T17:14:49,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/c808dc00bf4e4c5aa6a1eca1525e1e80 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c808dc00bf4e4c5aa6a1eca1525e1e80 2024-12-02T17:14:49,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c808dc00bf4e4c5aa6a1eca1525e1e80, entries=150, sequenceid=786, filesize=12.0 K 2024-12-02T17:14:49,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/e38dab56e33b448a85987a79b5b6accb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e38dab56e33b448a85987a79b5b6accb 2024-12-02T17:14:49,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e38dab56e33b448a85987a79b5b6accb, entries=150, sequenceid=786, filesize=12.0 K 2024-12-02T17:14:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for dcecfb16214ccdbe1c27643e3ed0b42c in 208ms, sequenceid=786, compaction requested=false 2024-12-02T17:14:49,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:49,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:49,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:49,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:49,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/8d11b9d0bf6c4a87b197e691492d5977 is 50, key is test_row_0/A:col10/1733159688917/Put/seqid=0 2024-12-02T17:14:49,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741954_1130 (size=14741) 2024-12-02T17:14:49,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=800 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/8d11b9d0bf6c4a87b197e691492d5977 2024-12-02T17:14:49,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159689298, max timestamp=1733159689314Num uniques 859; 2024-12-02T17:14:49,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:49,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159689298, max timestamp=1733159689314Num uniques 859; 2024-12-02T17:14:49,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:49,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159689298, max timestamp=1733159689314Num uniques 859; 2024-12-02T17:14:49,326 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:14:49,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/33efe224d30b4677a76e30c6e8f209a5 is 50, key is test_row_0/B:col10/1733159688917/Put/seqid=0 2024-12-02T17:14:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159749367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741955_1131 (size=12301) 2024-12-02T17:14:49,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159749371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159749373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:49,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:49,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:49,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159749477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159749497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159749498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:49,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35602 deadline: 1733159749656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,665 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159749691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159749707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159749717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=800 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/33efe224d30b4677a76e30c6e8f209a5 2024-12-02T17:14:49,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/36dd772fdd6b458c97412dd03448036d is 50, key is test_row_0/C:col10/1733159688917/Put/seqid=0 2024-12-02T17:14:49,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741956_1132 (size=12301) 2024-12-02T17:14:49,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35644 deadline: 1733159749892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,897 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in dcecfb16214ccdbe1c27643e3ed0b42c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:14:49,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:49,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:49,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:49,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159749997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159750026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159750026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:50,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:50,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:14:50,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=800 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/36dd772fdd6b458c97412dd03448036d 2024-12-02T17:14:50,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/8d11b9d0bf6c4a87b197e691492d5977 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d11b9d0bf6c4a87b197e691492d5977 2024-12-02T17:14:50,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d11b9d0bf6c4a87b197e691492d5977, entries=200, sequenceid=800, filesize=14.4 K 2024-12-02T17:14:50,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/33efe224d30b4677a76e30c6e8f209a5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33efe224d30b4677a76e30c6e8f209a5 2024-12-02T17:14:50,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33efe224d30b4677a76e30c6e8f209a5, entries=150, sequenceid=800, filesize=12.0 K 2024-12-02T17:14:50,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/36dd772fdd6b458c97412dd03448036d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/36dd772fdd6b458c97412dd03448036d 2024-12-02T17:14:50,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/36dd772fdd6b458c97412dd03448036d, entries=150, sequenceid=800, filesize=12.0 K 2024-12-02T17:14:50,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dcecfb16214ccdbe1c27643e3ed0b42c in 991ms, sequenceid=800, compaction requested=true 2024-12-02T17:14:50,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:50,253 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:50,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:50,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:50,254 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:50,255 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:50,255 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:50,255 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,255 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7dd507c7fbe549299970dd5067928d39, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ef7431ba04404ba78a091239aecf7757, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d11b9d0bf6c4a87b197e691492d5977] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=42.0 K 2024-12-02T17:14:50,255 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:50,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:50,256 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dd507c7fbe549299970dd5067928d39, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733159688041 2024-12-02T17:14:50,256 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/35594f8161604d11ac298123142e7f96, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c808dc00bf4e4c5aa6a1eca1525e1e80, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33efe224d30b4677a76e30c6e8f209a5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.3 K 2024-12-02T17:14:50,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7431ba04404ba78a091239aecf7757, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=786, earliestPutTs=1733159688768 2024-12-02T17:14:50,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 35594f8161604d11ac298123142e7f96, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733159688041 2024-12-02T17:14:50,257 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d11b9d0bf6c4a87b197e691492d5977, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1733159688917 2024-12-02T17:14:50,257 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c808dc00bf4e4c5aa6a1eca1525e1e80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=786, earliestPutTs=1733159688768 2024-12-02T17:14:50,258 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 33efe224d30b4677a76e30c6e8f209a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1733159688917 2024-12-02T17:14:50,267 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:50,268 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/d5e4a1cdb2d74bf4b9ed78dc102eb811 is 50, key is test_row_0/A:col10/1733159688917/Put/seqid=0 2024-12-02T17:14:50,278 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:50,279 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/983268ad47934df0b3d7dab055ff87d6 is 50, key is test_row_0/B:col10/1733159688917/Put/seqid=0 2024-12-02T17:14:50,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741957_1133 (size=13663) 2024-12-02T17:14:50,290 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/d5e4a1cdb2d74bf4b9ed78dc102eb811 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d5e4a1cdb2d74bf4b9ed78dc102eb811 2024-12-02T17:14:50,301 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into d5e4a1cdb2d74bf4b9ed78dc102eb811(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:50,301 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:50,301 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159690253; duration=0sec 2024-12-02T17:14:50,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741958_1134 (size=13663) 2024-12-02T17:14:50,301 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:50,301 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:50,301 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:50,305 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:50,305 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:50,306 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,306 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/fc12086a811f487c9046eabe7e331144, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e38dab56e33b448a85987a79b5b6accb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/36dd772fdd6b458c97412dd03448036d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.3 K 2024-12-02T17:14:50,307 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc12086a811f487c9046eabe7e331144, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=750, earliestPutTs=1733159688041 2024-12-02T17:14:50,308 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38dab56e33b448a85987a79b5b6accb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=786, earliestPutTs=1733159688768 2024-12-02T17:14:50,310 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36dd772fdd6b458c97412dd03448036d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1733159688917 2024-12-02T17:14:50,310 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/983268ad47934df0b3d7dab055ff87d6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/983268ad47934df0b3d7dab055ff87d6 2024-12-02T17:14:50,321 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:50,321 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/dc487a4e16ef4faeb3dd368314e51455 is 50, key is test_row_0/C:col10/1733159688917/Put/seqid=0 2024-12-02T17:14:50,322 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 983268ad47934df0b3d7dab055ff87d6(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:50,322 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:50,322 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159690253; duration=0sec 2024-12-02T17:14:50,322 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:50,322 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:50,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741959_1135 (size=13663) 2024-12-02T17:14:50,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:50,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:14:50,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/32a782cd7a2347e58090587313c9f722 is 50, key is test_row_0/A:col10/1733159689365/Put/seqid=0 2024-12-02T17:14:50,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741960_1136 (size=12301) 2024-12-02T17:14:50,385 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/32a782cd7a2347e58090587313c9f722 2024-12-02T17:14:50,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2a70333b98e340d6a5e46292800ec4c9 is 50, key is test_row_0/B:col10/1733159689365/Put/seqid=0 2024-12-02T17:14:50,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741961_1137 (size=12301) 2024-12-02T17:14:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:50,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. as already flushing 2024-12-02T17:14:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35624 deadline: 1733159750547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159750548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159750551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159750658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159750667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,749 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/dc487a4e16ef4faeb3dd368314e51455 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/dc487a4e16ef4faeb3dd368314e51455 2024-12-02T17:14:50,764 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into dc487a4e16ef4faeb3dd368314e51455(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:50,764 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:50,764 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159690254; duration=0sec 2024-12-02T17:14:50,764 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:50,764 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:50,786 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-02T17:14:50,786 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-02T17:14:50,786 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:50,786 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:50,787 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-02T17:14:50,787 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:50,788 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-02T17:14:50,788 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:50,791 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,791 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,791 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,791 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:50,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:50,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:50,792 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:50,792 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,792 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,792 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:50,856 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2a70333b98e340d6a5e46292800ec4c9 2024-12-02T17:14:50,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/7042dcc4faae42ac85e1da6a86903cf0 is 50, key is test_row_0/C:col10/1733159689365/Put/seqid=0 2024-12-02T17:14:50,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741962_1138 (size=12301) 2024-12-02T17:14:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159750877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159750877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:51,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:51,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:14:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1733159751187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35636 deadline: 1733159751187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:14:51,268 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=836 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/7042dcc4faae42ac85e1da6a86903cf0 2024-12-02T17:14:51,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/32a782cd7a2347e58090587313c9f722 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/32a782cd7a2347e58090587313c9f722 2024-12-02T17:14:51,277 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/32a782cd7a2347e58090587313c9f722, entries=150, sequenceid=836, filesize=12.0 K 2024-12-02T17:14:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/2a70333b98e340d6a5e46292800ec4c9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2a70333b98e340d6a5e46292800ec4c9 2024-12-02T17:14:51,283 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2a70333b98e340d6a5e46292800ec4c9, entries=150, sequenceid=836, filesize=12.0 K 2024-12-02T17:14:51,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/7042dcc4faae42ac85e1da6a86903cf0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7042dcc4faae42ac85e1da6a86903cf0 2024-12-02T17:14:51,288 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7042dcc4faae42ac85e1da6a86903cf0, entries=150, sequenceid=836, filesize=12.0 K 2024-12-02T17:14:51,288 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for dcecfb16214ccdbe1c27643e3ed0b42c in 924ms, sequenceid=836, compaction requested=false 2024-12-02T17:14:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:51,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-02T17:14:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-02T17:14:51,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T17:14:51,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4930 sec 2024-12-02T17:14:51,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.5030 sec 2024-12-02T17:14:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:14:51,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:14:51,569 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-02T17:14:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:14:51,569 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:51,569 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:14:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:14:51,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:14:51,569 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:51,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:51,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/bf995af7e6bd44bb8fe40c39dffadfec is 50, key is test_row_0/A:col10/1733159690547/Put/seqid=0 2024-12-02T17:14:51,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741963_1139 (size=12301) 2024-12-02T17:14:51,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=849 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/bf995af7e6bd44bb8fe40c39dffadfec 2024-12-02T17:14:51,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/405dd29d9e014442b6d13f9a1ed08ddb is 50, key is test_row_0/B:col10/1733159690547/Put/seqid=0 2024-12-02T17:14:51,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741964_1140 (size=12301) 2024-12-02T17:14:51,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=849 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/405dd29d9e014442b6d13f9a1ed08ddb 2024-12-02T17:14:51,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/f28a93a70fcd405b8f0abc93c9988121 is 50, key is test_row_0/C:col10/1733159690547/Put/seqid=0 2024-12-02T17:14:51,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741965_1141 (size=12301) 2024-12-02T17:14:51,700 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-02T17:14:51,700 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-02T17:14:51,700 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:51,700 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:51,700 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:51,700 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:51,700 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:51,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:51,700 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:51,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:52,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=849 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/f28a93a70fcd405b8f0abc93c9988121 2024-12-02T17:14:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/bf995af7e6bd44bb8fe40c39dffadfec as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bf995af7e6bd44bb8fe40c39dffadfec 2024-12-02T17:14:52,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bf995af7e6bd44bb8fe40c39dffadfec, entries=150, sequenceid=849, filesize=12.0 K 2024-12-02T17:14:52,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/405dd29d9e014442b6d13f9a1ed08ddb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/405dd29d9e014442b6d13f9a1ed08ddb 2024-12-02T17:14:52,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/405dd29d9e014442b6d13f9a1ed08ddb, entries=150, sequenceid=849, filesize=12.0 K 2024-12-02T17:14:52,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/f28a93a70fcd405b8f0abc93c9988121 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/f28a93a70fcd405b8f0abc93c9988121 2024-12-02T17:14:52,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/f28a93a70fcd405b8f0abc93c9988121, entries=150, sequenceid=849, filesize=12.0 K 2024-12-02T17:14:52,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for dcecfb16214ccdbe1c27643e3ed0b42c in 464ms, sequenceid=849, compaction requested=true 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:14:52,034 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dcecfb16214ccdbe1c27643e3ed0b42c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:14:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:52,034 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:52,035 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:52,035 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:52,035 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/B is initiating minor compaction (all files) 2024-12-02T17:14:52,035 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/A is initiating minor compaction (all files) 2024-12-02T17:14:52,035 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/B in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:52,035 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/A in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:52,035 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/983268ad47934df0b3d7dab055ff87d6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2a70333b98e340d6a5e46292800ec4c9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/405dd29d9e014442b6d13f9a1ed08ddb] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.4 K 2024-12-02T17:14:52,035 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d5e4a1cdb2d74bf4b9ed78dc102eb811, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/32a782cd7a2347e58090587313c9f722, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bf995af7e6bd44bb8fe40c39dffadfec] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.4 K 2024-12-02T17:14:52,036 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 983268ad47934df0b3d7dab055ff87d6, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1733159688917 2024-12-02T17:14:52,036 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5e4a1cdb2d74bf4b9ed78dc102eb811, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1733159688917 2024-12-02T17:14:52,036 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a70333b98e340d6a5e46292800ec4c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733159689365 2024-12-02T17:14:52,036 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a782cd7a2347e58090587313c9f722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733159689365 2024-12-02T17:14:52,036 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 405dd29d9e014442b6d13f9a1ed08ddb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=849, earliestPutTs=1733159690540 2024-12-02T17:14:52,037 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf995af7e6bd44bb8fe40c39dffadfec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=849, earliestPutTs=1733159690540 2024-12-02T17:14:52,045 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:52,046 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/9c617af59db34e12ac0d0a1e0a757041 is 50, key is test_row_0/B:col10/1733159690547/Put/seqid=0 2024-12-02T17:14:52,047 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#A#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:52,047 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/6ddcfe13a2164481aa7eaeb7bde86ec1 is 50, key is test_row_0/A:col10/1733159690547/Put/seqid=0 2024-12-02T17:14:52,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741966_1142 (size=13765) 2024-12-02T17:14:52,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741967_1143 (size=13765) 2024-12-02T17:14:52,459 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/9c617af59db34e12ac0d0a1e0a757041 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/9c617af59db34e12ac0d0a1e0a757041 2024-12-02T17:14:52,463 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/6ddcfe13a2164481aa7eaeb7bde86ec1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/6ddcfe13a2164481aa7eaeb7bde86ec1 2024-12-02T17:14:52,464 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/B of dcecfb16214ccdbe1c27643e3ed0b42c into 9c617af59db34e12ac0d0a1e0a757041(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:52,464 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:52,464 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/B, priority=13, startTime=1733159692034; duration=0sec 2024-12-02T17:14:52,465 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:14:52,465 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:B 2024-12-02T17:14:52,465 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:14:52,466 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:14:52,466 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): dcecfb16214ccdbe1c27643e3ed0b42c/C is initiating minor compaction (all files) 2024-12-02T17:14:52,466 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dcecfb16214ccdbe1c27643e3ed0b42c/C in TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:14:52,466 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/dc487a4e16ef4faeb3dd368314e51455, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7042dcc4faae42ac85e1da6a86903cf0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/f28a93a70fcd405b8f0abc93c9988121] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp, totalSize=37.4 K 2024-12-02T17:14:52,466 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting dc487a4e16ef4faeb3dd368314e51455, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=800, earliestPutTs=1733159688917 2024-12-02T17:14:52,467 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 7042dcc4faae42ac85e1da6a86903cf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=836, earliestPutTs=1733159689365 2024-12-02T17:14:52,467 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting f28a93a70fcd405b8f0abc93c9988121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=849, earliestPutTs=1733159690540 2024-12-02T17:14:52,469 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/A of dcecfb16214ccdbe1c27643e3ed0b42c into 6ddcfe13a2164481aa7eaeb7bde86ec1(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:52,469 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:52,469 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/A, priority=13, startTime=1733159692034; duration=0sec 2024-12-02T17:14:52,469 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:52,469 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:A 2024-12-02T17:14:52,477 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dcecfb16214ccdbe1c27643e3ed0b42c#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:14:52,478 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/106c1b48b64b4e13b6a95347cc254d8e is 50, key is test_row_0/C:col10/1733159690547/Put/seqid=0 2024-12-02T17:14:52,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741968_1144 (size=13765) 2024-12-02T17:14:52,889 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/106c1b48b64b4e13b6a95347cc254d8e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/106c1b48b64b4e13b6a95347cc254d8e 2024-12-02T17:14:52,899 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dcecfb16214ccdbe1c27643e3ed0b42c/C of dcecfb16214ccdbe1c27643e3ed0b42c into 106c1b48b64b4e13b6a95347cc254d8e(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:14:52,899 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dcecfb16214ccdbe1c27643e3ed0b42c: 2024-12-02T17:14:52,900 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., storeName=dcecfb16214ccdbe1c27643e3ed0b42c/C, priority=13, startTime=1733159692034; duration=0sec 2024-12-02T17:14:52,900 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:14:52,900 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dcecfb16214ccdbe1c27643e3ed0b42c:C 2024-12-02T17:14:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T17:14:52,987 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:14:53,709 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-02T17:14:53,709 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:53,709 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:53,709 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:53,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:54,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T17:14:59,960 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-02T17:14:59,960 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:14:59,960 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:59,960 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:59,960 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 150 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 192 2024-12-02T17:14:59,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1645 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1642 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 794 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2381 rows 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 792 2024-12-02T17:14:59,961 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2375 rows 2024-12-02T17:14:59,961 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:14:59,961 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:14:59,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:59,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:14:59,961 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:14:59,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T17:14:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T17:14:59,975 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159699975"}]},"ts":"1733159699975"} 2024-12-02T17:14:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T17:14:59,977 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T17:14:59,977 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T17:14:59,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:14:59,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, UNASSIGN}] 2024-12-02T17:14:59,985 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, UNASSIGN 2024-12-02T17:14:59,987 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=dcecfb16214ccdbe1c27643e3ed0b42c, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:14:59,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, UNASSIGN because future has completed 2024-12-02T17:14:59,990 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:14:59,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:15:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T17:15:00,147 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:15:00,147 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing dcecfb16214ccdbe1c27643e3ed0b42c, disabling compactions & flushes 2024-12-02T17:15:00,148 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. after waiting 0 ms 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:15:00,148 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing dcecfb16214ccdbe1c27643e3ed0b42c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=A 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=B 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dcecfb16214ccdbe1c27643e3ed0b42c, store=C 2024-12-02T17:15:00,148 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:00,152 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/4fcaa57d2e80421a8eb0b9b49b4f4272 is 50, key is test_row_0/A:col10/1733159691698/Put/seqid=0 2024-12-02T17:15:00,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741969_1145 (size=12301) 2024-12-02T17:15:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T17:15:00,558 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=859 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/4fcaa57d2e80421a8eb0b9b49b4f4272 2024-12-02T17:15:00,566 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/3a3795fb518547a8b4106ecf0a4c0f88 is 50, key is test_row_0/B:col10/1733159691698/Put/seqid=0 2024-12-02T17:15:00,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741970_1146 (size=12301) 2024-12-02T17:15:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T17:15:00,971 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=859 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/3a3795fb518547a8b4106ecf0a4c0f88 2024-12-02T17:15:00,984 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/ae04a130041c4dfd948261661d5310e8 is 50, key is test_row_0/C:col10/1733159691698/Put/seqid=0 2024-12-02T17:15:00,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741971_1147 (size=12301) 2024-12-02T17:15:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T17:15:01,395 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=859 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/ae04a130041c4dfd948261661d5310e8 2024-12-02T17:15:01,405 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/A/4fcaa57d2e80421a8eb0b9b49b4f4272 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4fcaa57d2e80421a8eb0b9b49b4f4272 2024-12-02T17:15:01,413 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4fcaa57d2e80421a8eb0b9b49b4f4272, entries=150, sequenceid=859, filesize=12.0 K 2024-12-02T17:15:01,417 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/B/3a3795fb518547a8b4106ecf0a4c0f88 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/3a3795fb518547a8b4106ecf0a4c0f88 2024-12-02T17:15:01,432 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/3a3795fb518547a8b4106ecf0a4c0f88, entries=150, sequenceid=859, filesize=12.0 K 2024-12-02T17:15:01,434 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/.tmp/C/ae04a130041c4dfd948261661d5310e8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ae04a130041c4dfd948261661d5310e8 2024-12-02T17:15:01,440 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ae04a130041c4dfd948261661d5310e8, entries=150, sequenceid=859, filesize=12.0 K 2024-12-02T17:15:01,442 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for dcecfb16214ccdbe1c27643e3ed0b42c in 1294ms, sequenceid=859, compaction requested=false 2024-12-02T17:15:01,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/c4dc4a0d2019472ca087aae7afe76ee4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ac911b2598514bf89bc27328464f9626, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae3d863d577342bf80ad575ab6415a21, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/35c373df4701459cbc9cd775916a4886, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/eac15d4ba601488381aaf374c61c387b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/14a685c335034110b01d4f9c82a5dd3e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/e37d94306ca3487eb0f966e607e3628f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/49c9103387544b1ea06730caa72a02c8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b137837d41964becadbdc5b0923e8eef, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4faef52d83034ead9e5f4053f3a965a0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/1aabda36f8cb4b8f8e9a8d98e012b41c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cff43c427e3f452f8131c2d4ca73f10c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b59f7160d48442fe8da68c9716f9d71a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7329024ddf574ed58b8e29f406400b75, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f6df152254c04b328072c39e03bb8a99, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ca6bfa0e867b49daa32ebb77fd173de2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d2e0a2735fd34348aaaaf64e3409d311, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/3c4249cd8a50488ba73d2fab53c46283, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b8d0e52b5d524619a19b39ee34ea0e94, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae5d90a894e0436e88d5421758101c5b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0b46c3325de64aacb57678f5c274102d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/9e791d7e432e48b3b834baffb9cc2518, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a6de56ade38a44e899fe278ac0d89a0c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cbc15b8461d7434a810eb39c7a0e0b1f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/629004400a0443eb8c9b58f9113321b2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d647d042fe545dc837ccd3d90781e26, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/2894bd25d1c240478d18188bbaebf7a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ff6f087042f74034ab88da9c50a43024, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39a7e3b020454c6cb91f904186fba16e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/82943e60d5c34d24b54ee8462eec19d0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/79d902738e4a4a9183fc0b57b3a26f30, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bde65ae89df5410497af2f33989850b0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a18ad75b77fc42679df84bff846a6d5b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39afe24997b246bab2d8aa11f7a5f04f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/163d5218f2bc4b3888d252709e14ee92, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f107302be3d7409db5b3b1e01d7af595, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0d0ff3bc34264dda83f5d9bcb7b6b25f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7dd507c7fbe549299970dd5067928d39, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ef7431ba04404ba78a091239aecf7757, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d11b9d0bf6c4a87b197e691492d5977, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d5e4a1cdb2d74bf4b9ed78dc102eb811, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/32a782cd7a2347e58090587313c9f722, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bf995af7e6bd44bb8fe40c39dffadfec] to archive 2024-12-02T17:15:01,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:15:01,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/c4dc4a0d2019472ca087aae7afe76ee4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/c4dc4a0d2019472ca087aae7afe76ee4 2024-12-02T17:15:01,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ac911b2598514bf89bc27328464f9626 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ac911b2598514bf89bc27328464f9626 2024-12-02T17:15:01,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae3d863d577342bf80ad575ab6415a21 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae3d863d577342bf80ad575ab6415a21 2024-12-02T17:15:01,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/35c373df4701459cbc9cd775916a4886 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/35c373df4701459cbc9cd775916a4886 2024-12-02T17:15:01,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/eac15d4ba601488381aaf374c61c387b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/eac15d4ba601488381aaf374c61c387b 2024-12-02T17:15:01,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/14a685c335034110b01d4f9c82a5dd3e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/14a685c335034110b01d4f9c82a5dd3e 2024-12-02T17:15:01,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/e37d94306ca3487eb0f966e607e3628f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/e37d94306ca3487eb0f966e607e3628f 2024-12-02T17:15:01,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/49c9103387544b1ea06730caa72a02c8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/49c9103387544b1ea06730caa72a02c8 2024-12-02T17:15:01,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b137837d41964becadbdc5b0923e8eef to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b137837d41964becadbdc5b0923e8eef 2024-12-02T17:15:01,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4faef52d83034ead9e5f4053f3a965a0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4faef52d83034ead9e5f4053f3a965a0 2024-12-02T17:15:01,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/1aabda36f8cb4b8f8e9a8d98e012b41c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/1aabda36f8cb4b8f8e9a8d98e012b41c 2024-12-02T17:15:01,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cff43c427e3f452f8131c2d4ca73f10c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cff43c427e3f452f8131c2d4ca73f10c 2024-12-02T17:15:01,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b59f7160d48442fe8da68c9716f9d71a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b59f7160d48442fe8da68c9716f9d71a 2024-12-02T17:15:01,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7329024ddf574ed58b8e29f406400b75 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7329024ddf574ed58b8e29f406400b75 2024-12-02T17:15:01,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f6df152254c04b328072c39e03bb8a99 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f6df152254c04b328072c39e03bb8a99 2024-12-02T17:15:01,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ca6bfa0e867b49daa32ebb77fd173de2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ca6bfa0e867b49daa32ebb77fd173de2 2024-12-02T17:15:01,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d2e0a2735fd34348aaaaf64e3409d311 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d2e0a2735fd34348aaaaf64e3409d311 2024-12-02T17:15:01,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/3c4249cd8a50488ba73d2fab53c46283 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/3c4249cd8a50488ba73d2fab53c46283 2024-12-02T17:15:01,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b8d0e52b5d524619a19b39ee34ea0e94 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/b8d0e52b5d524619a19b39ee34ea0e94 2024-12-02T17:15:01,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae5d90a894e0436e88d5421758101c5b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ae5d90a894e0436e88d5421758101c5b 2024-12-02T17:15:01,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0b46c3325de64aacb57678f5c274102d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0b46c3325de64aacb57678f5c274102d 2024-12-02T17:15:01,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/9e791d7e432e48b3b834baffb9cc2518 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/9e791d7e432e48b3b834baffb9cc2518 2024-12-02T17:15:01,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a6de56ade38a44e899fe278ac0d89a0c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a6de56ade38a44e899fe278ac0d89a0c 2024-12-02T17:15:01,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cbc15b8461d7434a810eb39c7a0e0b1f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/cbc15b8461d7434a810eb39c7a0e0b1f 2024-12-02T17:15:01,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/629004400a0443eb8c9b58f9113321b2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/629004400a0443eb8c9b58f9113321b2 2024-12-02T17:15:01,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d647d042fe545dc837ccd3d90781e26 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d647d042fe545dc837ccd3d90781e26 2024-12-02T17:15:01,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/2894bd25d1c240478d18188bbaebf7a8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/2894bd25d1c240478d18188bbaebf7a8 2024-12-02T17:15:01,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ff6f087042f74034ab88da9c50a43024 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ff6f087042f74034ab88da9c50a43024 2024-12-02T17:15:01,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39a7e3b020454c6cb91f904186fba16e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39a7e3b020454c6cb91f904186fba16e 2024-12-02T17:15:01,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/82943e60d5c34d24b54ee8462eec19d0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/82943e60d5c34d24b54ee8462eec19d0 2024-12-02T17:15:01,548 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/79d902738e4a4a9183fc0b57b3a26f30 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/79d902738e4a4a9183fc0b57b3a26f30 2024-12-02T17:15:01,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bde65ae89df5410497af2f33989850b0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bde65ae89df5410497af2f33989850b0 2024-12-02T17:15:01,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a18ad75b77fc42679df84bff846a6d5b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/a18ad75b77fc42679df84bff846a6d5b 2024-12-02T17:15:01,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39afe24997b246bab2d8aa11f7a5f04f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/39afe24997b246bab2d8aa11f7a5f04f 2024-12-02T17:15:01,555 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/163d5218f2bc4b3888d252709e14ee92 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/163d5218f2bc4b3888d252709e14ee92 2024-12-02T17:15:01,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f107302be3d7409db5b3b1e01d7af595 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/f107302be3d7409db5b3b1e01d7af595 2024-12-02T17:15:01,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0d0ff3bc34264dda83f5d9bcb7b6b25f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/0d0ff3bc34264dda83f5d9bcb7b6b25f 2024-12-02T17:15:01,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7dd507c7fbe549299970dd5067928d39 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/7dd507c7fbe549299970dd5067928d39 2024-12-02T17:15:01,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ef7431ba04404ba78a091239aecf7757 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/ef7431ba04404ba78a091239aecf7757 2024-12-02T17:15:01,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d11b9d0bf6c4a87b197e691492d5977 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/8d11b9d0bf6c4a87b197e691492d5977 2024-12-02T17:15:01,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d5e4a1cdb2d74bf4b9ed78dc102eb811 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/d5e4a1cdb2d74bf4b9ed78dc102eb811 2024-12-02T17:15:01,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/32a782cd7a2347e58090587313c9f722 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/32a782cd7a2347e58090587313c9f722 2024-12-02T17:15:01,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bf995af7e6bd44bb8fe40c39dffadfec to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/bf995af7e6bd44bb8fe40c39dffadfec 2024-12-02T17:15:01,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2aa30f07eaba49cea47a02815ab4185c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/4423e45c383f485a8b87f79076c7de7d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/07ffc256e7f94541a436d0bd93275e02, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/ff0bf45030fb4cf8b3a523c1f991dcad, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2237d6fcb1194d2d825f420b00d0ee4f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/69f13fd2752847bfad9e80c04d775960, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/de7fde781b784f239f071c49b505457c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/731ce4f75b664f6cb51289958f3105d2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/46be562888b842ccbdd7441eb70ae409, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d65ee3866107402eb47a9936ff684e19, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e92e4387ed644052a39b3879af771b7b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f38ac2a08af94b50a72380446f98b42a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c0175f97f86a488e936e61c85f1bbaf9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/55feb41fbd4941ccbef1a8668aea4c35, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/47cf69df0ba04f57aee1fb49b8823371, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/05a1915fae534465a5131fa336e28093, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c6c586ac31044026b19cc94495dda540, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33acd482ed3f4915afd878c4e2ce78b5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/79771cb38710482881ccea0bc5898fa9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/230d3abc565a41e0a80ee0b1ab7b8509, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/dda3d332094240e197eff4b6938ee730, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e600f468b9fc455dba93ed17b45bec2d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5a5980c80dc242ce81546a26385f081d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/0784a1bbcf9d4400b7322195be734678, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5f03440ea32f40c38913e1926bdaabe1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2d962c1c03d9497ea42411b69a66a34a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/b47043bef4164373a1a8e433f86dd452, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f18c4f5449b14f3889fb65fc3cbf2bb1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d21027167d874c008b66a4c5cf2514e9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5014f63a77ba4445a86b2b9197283dbf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2fdc2ee19eec457f8ab3d064e94bc39f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d112d041b1af4531a14bcca27de8baf8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/44d19b6a403348ea8b05202ed7eed96f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/a4ff7ac8fd8e42edac5fd7805f4d3455, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/34cb23375b8c4133a3f874e54ca407e5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2445be97285e40ea9b4bb7a7d2f1bdf2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/35594f8161604d11ac298123142e7f96, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2cdc9a1ad0d0443880b35d77849d29d5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c808dc00bf4e4c5aa6a1eca1525e1e80, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/983268ad47934df0b3d7dab055ff87d6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33efe224d30b4677a76e30c6e8f209a5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2a70333b98e340d6a5e46292800ec4c9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/405dd29d9e014442b6d13f9a1ed08ddb] to archive 2024-12-02T17:15:01,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:15:01,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2aa30f07eaba49cea47a02815ab4185c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2aa30f07eaba49cea47a02815ab4185c 2024-12-02T17:15:01,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/4423e45c383f485a8b87f79076c7de7d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/4423e45c383f485a8b87f79076c7de7d 2024-12-02T17:15:01,595 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/07ffc256e7f94541a436d0bd93275e02 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/07ffc256e7f94541a436d0bd93275e02 2024-12-02T17:15:01,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/ff0bf45030fb4cf8b3a523c1f991dcad to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/ff0bf45030fb4cf8b3a523c1f991dcad 2024-12-02T17:15:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2237d6fcb1194d2d825f420b00d0ee4f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2237d6fcb1194d2d825f420b00d0ee4f 2024-12-02T17:15:01,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/69f13fd2752847bfad9e80c04d775960 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/69f13fd2752847bfad9e80c04d775960 2024-12-02T17:15:01,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/de7fde781b784f239f071c49b505457c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/de7fde781b784f239f071c49b505457c 2024-12-02T17:15:01,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/731ce4f75b664f6cb51289958f3105d2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/731ce4f75b664f6cb51289958f3105d2 2024-12-02T17:15:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/46be562888b842ccbdd7441eb70ae409 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/46be562888b842ccbdd7441eb70ae409 2024-12-02T17:15:01,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d65ee3866107402eb47a9936ff684e19 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d65ee3866107402eb47a9936ff684e19 2024-12-02T17:15:01,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e92e4387ed644052a39b3879af771b7b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e92e4387ed644052a39b3879af771b7b 2024-12-02T17:15:01,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f38ac2a08af94b50a72380446f98b42a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f38ac2a08af94b50a72380446f98b42a 2024-12-02T17:15:01,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c0175f97f86a488e936e61c85f1bbaf9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c0175f97f86a488e936e61c85f1bbaf9 2024-12-02T17:15:01,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/55feb41fbd4941ccbef1a8668aea4c35 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/55feb41fbd4941ccbef1a8668aea4c35 2024-12-02T17:15:01,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/47cf69df0ba04f57aee1fb49b8823371 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/47cf69df0ba04f57aee1fb49b8823371 2024-12-02T17:15:01,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/05a1915fae534465a5131fa336e28093 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/05a1915fae534465a5131fa336e28093 2024-12-02T17:15:01,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c6c586ac31044026b19cc94495dda540 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c6c586ac31044026b19cc94495dda540 2024-12-02T17:15:01,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33acd482ed3f4915afd878c4e2ce78b5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33acd482ed3f4915afd878c4e2ce78b5 2024-12-02T17:15:01,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/79771cb38710482881ccea0bc5898fa9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/79771cb38710482881ccea0bc5898fa9 2024-12-02T17:15:01,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/230d3abc565a41e0a80ee0b1ab7b8509 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/230d3abc565a41e0a80ee0b1ab7b8509 2024-12-02T17:15:01,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/dda3d332094240e197eff4b6938ee730 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/dda3d332094240e197eff4b6938ee730 2024-12-02T17:15:01,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e600f468b9fc455dba93ed17b45bec2d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/e600f468b9fc455dba93ed17b45bec2d 2024-12-02T17:15:01,626 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5a5980c80dc242ce81546a26385f081d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5a5980c80dc242ce81546a26385f081d 2024-12-02T17:15:01,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/0784a1bbcf9d4400b7322195be734678 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/0784a1bbcf9d4400b7322195be734678 2024-12-02T17:15:01,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5f03440ea32f40c38913e1926bdaabe1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5f03440ea32f40c38913e1926bdaabe1 2024-12-02T17:15:01,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2d962c1c03d9497ea42411b69a66a34a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2d962c1c03d9497ea42411b69a66a34a 2024-12-02T17:15:01,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/b47043bef4164373a1a8e433f86dd452 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/b47043bef4164373a1a8e433f86dd452 2024-12-02T17:15:01,633 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f18c4f5449b14f3889fb65fc3cbf2bb1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/f18c4f5449b14f3889fb65fc3cbf2bb1 2024-12-02T17:15:01,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d21027167d874c008b66a4c5cf2514e9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d21027167d874c008b66a4c5cf2514e9 2024-12-02T17:15:01,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5014f63a77ba4445a86b2b9197283dbf to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/5014f63a77ba4445a86b2b9197283dbf 2024-12-02T17:15:01,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2fdc2ee19eec457f8ab3d064e94bc39f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2fdc2ee19eec457f8ab3d064e94bc39f 2024-12-02T17:15:01,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d112d041b1af4531a14bcca27de8baf8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/d112d041b1af4531a14bcca27de8baf8 2024-12-02T17:15:01,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/44d19b6a403348ea8b05202ed7eed96f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/44d19b6a403348ea8b05202ed7eed96f 2024-12-02T17:15:01,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/a4ff7ac8fd8e42edac5fd7805f4d3455 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/a4ff7ac8fd8e42edac5fd7805f4d3455 2024-12-02T17:15:01,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/34cb23375b8c4133a3f874e54ca407e5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/34cb23375b8c4133a3f874e54ca407e5 2024-12-02T17:15:01,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2445be97285e40ea9b4bb7a7d2f1bdf2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2445be97285e40ea9b4bb7a7d2f1bdf2 2024-12-02T17:15:01,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/35594f8161604d11ac298123142e7f96 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/35594f8161604d11ac298123142e7f96 2024-12-02T17:15:01,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2cdc9a1ad0d0443880b35d77849d29d5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2cdc9a1ad0d0443880b35d77849d29d5 2024-12-02T17:15:01,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c808dc00bf4e4c5aa6a1eca1525e1e80 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/c808dc00bf4e4c5aa6a1eca1525e1e80 2024-12-02T17:15:01,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/983268ad47934df0b3d7dab055ff87d6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/983268ad47934df0b3d7dab055ff87d6 2024-12-02T17:15:01,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33efe224d30b4677a76e30c6e8f209a5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/33efe224d30b4677a76e30c6e8f209a5 2024-12-02T17:15:01,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2a70333b98e340d6a5e46292800ec4c9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/2a70333b98e340d6a5e46292800ec4c9 2024-12-02T17:15:01,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/405dd29d9e014442b6d13f9a1ed08ddb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/405dd29d9e014442b6d13f9a1ed08ddb 2024-12-02T17:15:01,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/241b3c4839e643ba961017a9595d842c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/5855d074785f4d268efeb00fb2faaf32, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1b823bdc0e7e46ae8ae782666504d8b7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fd42dd3ca32462db6ff94c41a3c3b7b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/6213b109cee94c47867052ee084905bd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/027765d4429348ee9a8cbd691d29c79e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/0eb433f6bd9145b886643eb3f34a8f88, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8afcd483a74e4b7ea2bcfc6c3efdd86d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/24a6b1ca93b84c069265edb762fdfeec, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/c613750a238f4c0b9df127bd7f940942, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e61e968db60647cd853c5bcd76a5d6eb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/b9489e54140a4028af601e9f40f70065, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/cd1ff723215a4b4c8218d3ea061e96c5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1c69998a8c14411b8347b0ccc38c4c65, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/65d0cd7010e34fde80e66713ecbc9def, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/847cb90e679a4f6b9ce3bd85621d3acf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9f3107d95ec9490291605bd4e67204dd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8811144d9cc046f88f304601b58c53bf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/97d890c33c87491f9fc4f8279f6ba6cc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9c779e90074c408892a21b2297fc562f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e656bb9d5b0e4ef88516ec79fec7b6ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2bf42a1192db4d97bf71b953bb8c2205, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7747f099ef1c40e1bd95a459f8c74251, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2b3d76bc7ba4455cb05d1538dcaca659, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a4a9d854dc56441ab5bca49e31f22dcf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a3ef27e43596452db7393f90e8e6582c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ba9b8763d64a4e5f865735880da4e7ed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/271ffe0ca7fd456d8da183c6a65beef4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e7613309e60540edaa5bd4ec1cfa175a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/44f5148824bb4dfbaff95937d1ec1b09, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fe1d697220340e3808fd6bee7c965bf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/bd12199a88dc4d9ca6a4b189abba0453, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1d24afa4cca54742a01dbb20dc6389d0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/060c40ec45c941dba22313637aec883e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/793981de67ba46cd8acf612ff16ea059, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/05b966642cb24817b8d62552d85c17f4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/fc12086a811f487c9046eabe7e331144, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/96ae971fc7d64f588851afe58ed150d5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e38dab56e33b448a85987a79b5b6accb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/dc487a4e16ef4faeb3dd368314e51455, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/36dd772fdd6b458c97412dd03448036d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7042dcc4faae42ac85e1da6a86903cf0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/f28a93a70fcd405b8f0abc93c9988121] to archive 2024-12-02T17:15:01,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:15:01,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/241b3c4839e643ba961017a9595d842c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/241b3c4839e643ba961017a9595d842c 2024-12-02T17:15:01,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/5855d074785f4d268efeb00fb2faaf32 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/5855d074785f4d268efeb00fb2faaf32 2024-12-02T17:15:01,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1b823bdc0e7e46ae8ae782666504d8b7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1b823bdc0e7e46ae8ae782666504d8b7 2024-12-02T17:15:01,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fd42dd3ca32462db6ff94c41a3c3b7b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fd42dd3ca32462db6ff94c41a3c3b7b 2024-12-02T17:15:01,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/6213b109cee94c47867052ee084905bd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/6213b109cee94c47867052ee084905bd 2024-12-02T17:15:01,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/027765d4429348ee9a8cbd691d29c79e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/027765d4429348ee9a8cbd691d29c79e 2024-12-02T17:15:01,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/0eb433f6bd9145b886643eb3f34a8f88 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/0eb433f6bd9145b886643eb3f34a8f88 2024-12-02T17:15:01,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8afcd483a74e4b7ea2bcfc6c3efdd86d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8afcd483a74e4b7ea2bcfc6c3efdd86d 2024-12-02T17:15:01,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/24a6b1ca93b84c069265edb762fdfeec to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/24a6b1ca93b84c069265edb762fdfeec 2024-12-02T17:15:01,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/c613750a238f4c0b9df127bd7f940942 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/c613750a238f4c0b9df127bd7f940942 2024-12-02T17:15:01,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e61e968db60647cd853c5bcd76a5d6eb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e61e968db60647cd853c5bcd76a5d6eb 2024-12-02T17:15:01,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/b9489e54140a4028af601e9f40f70065 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/b9489e54140a4028af601e9f40f70065 2024-12-02T17:15:01,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/cd1ff723215a4b4c8218d3ea061e96c5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/cd1ff723215a4b4c8218d3ea061e96c5 2024-12-02T17:15:01,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1c69998a8c14411b8347b0ccc38c4c65 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1c69998a8c14411b8347b0ccc38c4c65 2024-12-02T17:15:01,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/65d0cd7010e34fde80e66713ecbc9def to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/65d0cd7010e34fde80e66713ecbc9def 2024-12-02T17:15:01,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/847cb90e679a4f6b9ce3bd85621d3acf to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/847cb90e679a4f6b9ce3bd85621d3acf 2024-12-02T17:15:01,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9f3107d95ec9490291605bd4e67204dd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9f3107d95ec9490291605bd4e67204dd 2024-12-02T17:15:01,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8811144d9cc046f88f304601b58c53bf to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/8811144d9cc046f88f304601b58c53bf 2024-12-02T17:15:01,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/97d890c33c87491f9fc4f8279f6ba6cc to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/97d890c33c87491f9fc4f8279f6ba6cc 2024-12-02T17:15:01,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9c779e90074c408892a21b2297fc562f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/9c779e90074c408892a21b2297fc562f 2024-12-02T17:15:01,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e656bb9d5b0e4ef88516ec79fec7b6ea to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e656bb9d5b0e4ef88516ec79fec7b6ea 2024-12-02T17:15:01,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2bf42a1192db4d97bf71b953bb8c2205 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2bf42a1192db4d97bf71b953bb8c2205 2024-12-02T17:15:01,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7747f099ef1c40e1bd95a459f8c74251 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7747f099ef1c40e1bd95a459f8c74251 2024-12-02T17:15:01,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2b3d76bc7ba4455cb05d1538dcaca659 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/2b3d76bc7ba4455cb05d1538dcaca659 2024-12-02T17:15:01,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a4a9d854dc56441ab5bca49e31f22dcf to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a4a9d854dc56441ab5bca49e31f22dcf 2024-12-02T17:15:01,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a3ef27e43596452db7393f90e8e6582c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/a3ef27e43596452db7393f90e8e6582c 2024-12-02T17:15:01,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ba9b8763d64a4e5f865735880da4e7ed to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ba9b8763d64a4e5f865735880da4e7ed 2024-12-02T17:15:01,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/271ffe0ca7fd456d8da183c6a65beef4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/271ffe0ca7fd456d8da183c6a65beef4 2024-12-02T17:15:01,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e7613309e60540edaa5bd4ec1cfa175a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e7613309e60540edaa5bd4ec1cfa175a 2024-12-02T17:15:01,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/44f5148824bb4dfbaff95937d1ec1b09 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/44f5148824bb4dfbaff95937d1ec1b09 2024-12-02T17:15:01,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fe1d697220340e3808fd6bee7c965bf to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1fe1d697220340e3808fd6bee7c965bf 2024-12-02T17:15:01,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/bd12199a88dc4d9ca6a4b189abba0453 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/bd12199a88dc4d9ca6a4b189abba0453 2024-12-02T17:15:01,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1d24afa4cca54742a01dbb20dc6389d0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/1d24afa4cca54742a01dbb20dc6389d0 2024-12-02T17:15:01,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/060c40ec45c941dba22313637aec883e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/060c40ec45c941dba22313637aec883e 2024-12-02T17:15:01,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/793981de67ba46cd8acf612ff16ea059 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/793981de67ba46cd8acf612ff16ea059 2024-12-02T17:15:01,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/05b966642cb24817b8d62552d85c17f4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/05b966642cb24817b8d62552d85c17f4 2024-12-02T17:15:01,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/fc12086a811f487c9046eabe7e331144 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/fc12086a811f487c9046eabe7e331144 2024-12-02T17:15:01,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/96ae971fc7d64f588851afe58ed150d5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/96ae971fc7d64f588851afe58ed150d5 2024-12-02T17:15:01,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e38dab56e33b448a85987a79b5b6accb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/e38dab56e33b448a85987a79b5b6accb 2024-12-02T17:15:01,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/dc487a4e16ef4faeb3dd368314e51455 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/dc487a4e16ef4faeb3dd368314e51455 2024-12-02T17:15:01,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/36dd772fdd6b458c97412dd03448036d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/36dd772fdd6b458c97412dd03448036d 2024-12-02T17:15:01,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7042dcc4faae42ac85e1da6a86903cf0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/7042dcc4faae42ac85e1da6a86903cf0 2024-12-02T17:15:01,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/f28a93a70fcd405b8f0abc93c9988121 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/f28a93a70fcd405b8f0abc93c9988121 2024-12-02T17:15:01,746 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/recovered.edits/862.seqid, newMaxSeqId=862, maxSeqId=1 2024-12-02T17:15:01,747 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. 2024-12-02T17:15:01,747 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for dcecfb16214ccdbe1c27643e3ed0b42c: Waiting for close lock at 1733159700147Running coprocessor pre-close hooks at 1733159700148 (+1 ms)Disabling compacts and flushes for region at 1733159700148Disabling writes for close at 1733159700148Obtaining lock to block concurrent updates at 1733159700148Preparing flush snapshotting stores in dcecfb16214ccdbe1c27643e3ed0b42c at 1733159700148Finished memstore snapshotting TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733159700148Flushing stores of TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c. at 1733159700149 (+1 ms)Flushing dcecfb16214ccdbe1c27643e3ed0b42c/A: creating writer at 1733159700149Flushing dcecfb16214ccdbe1c27643e3ed0b42c/A: appending metadata at 1733159700152 (+3 ms)Flushing dcecfb16214ccdbe1c27643e3ed0b42c/A: closing flushed file at 1733159700152Flushing dcecfb16214ccdbe1c27643e3ed0b42c/B: creating writer at 1733159700563 (+411 ms)Flushing dcecfb16214ccdbe1c27643e3ed0b42c/B: appending metadata at 1733159700565 (+2 ms)Flushing dcecfb16214ccdbe1c27643e3ed0b42c/B: closing flushed file at 1733159700565Flushing dcecfb16214ccdbe1c27643e3ed0b42c/C: creating writer at 1733159700978 (+413 ms)Flushing dcecfb16214ccdbe1c27643e3ed0b42c/C: appending metadata at 1733159700984 (+6 ms)Flushing dcecfb16214ccdbe1c27643e3ed0b42c/C: closing flushed file at 1733159700984Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45d5ff6e: reopening flushed file at 1733159701402 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5da2300f: reopening flushed file at 1733159701413 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419e34eb: reopening flushed file at 1733159701432 (+19 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for dcecfb16214ccdbe1c27643e3ed0b42c in 1294ms, sequenceid=859, compaction requested=false at 1733159701442 (+10 ms)Writing region close event to WAL at 1733159701742 (+300 ms)Running coprocessor post-close hooks at 1733159701747 (+5 ms)Closed at 1733159701747 2024-12-02T17:15:01,750 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:15:01,751 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=dcecfb16214ccdbe1c27643e3ed0b42c, regionState=CLOSED 2024-12-02T17:15:01,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:15:01,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-02T17:15:01,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure dcecfb16214ccdbe1c27643e3ed0b42c, server=3b3375d1a925,40723,1733159666211 in 1.7710 sec 2024-12-02T17:15:01,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-02T17:15:01,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dcecfb16214ccdbe1c27643e3ed0b42c, UNASSIGN in 1.7800 sec 2024-12-02T17:15:01,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-02T17:15:01,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7890 sec 2024-12-02T17:15:01,772 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159701771"}]},"ts":"1733159701771"} 2024-12-02T17:15:01,773 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T17:15:01,773 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T17:15:01,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8060 sec 2024-12-02T17:15:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T17:15:02,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:02,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T17:15:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,139 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T17:15:02,140 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:15:02,150 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/recovered.edits] 2024-12-02T17:15:02,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4fcaa57d2e80421a8eb0b9b49b4f4272 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/4fcaa57d2e80421a8eb0b9b49b4f4272 2024-12-02T17:15:02,155 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/6ddcfe13a2164481aa7eaeb7bde86ec1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/A/6ddcfe13a2164481aa7eaeb7bde86ec1 2024-12-02T17:15:02,166 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/3a3795fb518547a8b4106ecf0a4c0f88 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/3a3795fb518547a8b4106ecf0a4c0f88 2024-12-02T17:15:02,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/9c617af59db34e12ac0d0a1e0a757041 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/B/9c617af59db34e12ac0d0a1e0a757041 2024-12-02T17:15:02,173 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/106c1b48b64b4e13b6a95347cc254d8e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/106c1b48b64b4e13b6a95347cc254d8e 2024-12-02T17:15:02,175 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ae04a130041c4dfd948261661d5310e8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/C/ae04a130041c4dfd948261661d5310e8 2024-12-02T17:15:02,179 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/recovered.edits/862.seqid to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c/recovered.edits/862.seqid 2024-12-02T17:15:02,180 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/dcecfb16214ccdbe1c27643e3ed0b42c 2024-12-02T17:15:02,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T17:15:02,191 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T17:15:02,201 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T17:15:02,237 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T17:15:02,238 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,238 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T17:15:02,238 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733159702238"}]},"ts":"9223372036854775807"} 2024-12-02T17:15:02,242 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T17:15:02,242 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dcecfb16214ccdbe1c27643e3ed0b42c, NAME => 'TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T17:15:02,243 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T17:15:02,243 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733159702243"}]},"ts":"9223372036854775807"} 2024-12-02T17:15:02,245 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T17:15:02,246 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-12-02T17:15:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T17:15:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-02T17:15:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T17:15:02,458 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:02,473 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=246 (was 210) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645024115_22 at /127.0.0.1:38686 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3b3375d1a925:40723-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=925 (was 855) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3137 (was 4076) 2024-12-02T17:15:02,489 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=246, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=925, ProcessCount=11, AvailableMemoryMB=3137 2024-12-02T17:15:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:15:02,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:15:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:02,494 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:15:02,494 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:02,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-02T17:15:02,495 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:15:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T17:15:02,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741972_1148 (size=1061) 2024-12-02T17:15:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T17:15:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T17:15:02,913 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:15:02,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741973_1149 (size=53) 2024-12-02T17:15:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T17:15:03,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:03,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 189a4a1fc2d783a754e60fdd039aa718, disabling compactions & flushes 2024-12-02T17:15:03,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. after waiting 0 ms 2024-12-02T17:15:03,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 189a4a1fc2d783a754e60fdd039aa718: Waiting for close lock at 1733159703330Disabling compacts and flushes for region at 1733159703330Disabling writes for close at 1733159703330Writing region close event to WAL at 1733159703330Closed at 1733159703330 2024-12-02T17:15:03,331 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:15:03,332 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733159703331"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159703331"}]},"ts":"1733159703331"} 2024-12-02T17:15:03,334 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T17:15:03,334 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:15:03,335 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159703335"}]},"ts":"1733159703335"} 2024-12-02T17:15:03,336 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T17:15:03,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, ASSIGN}] 2024-12-02T17:15:03,338 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, ASSIGN 2024-12-02T17:15:03,339 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, ASSIGN; state=OFFLINE, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=false 2024-12-02T17:15:03,489 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:03,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, ASSIGN because future has completed 2024-12-02T17:15:03,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:15:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T17:15:03,648 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,648 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:15:03,648 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,648 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:03,649 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,649 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,650 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,651 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:03,652 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 189a4a1fc2d783a754e60fdd039aa718 columnFamilyName A 2024-12-02T17:15:03,652 DEBUG [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:03,652 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(327): Store=189a4a1fc2d783a754e60fdd039aa718/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:03,653 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,654 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:03,654 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 189a4a1fc2d783a754e60fdd039aa718 columnFamilyName B 2024-12-02T17:15:03,654 DEBUG [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:03,654 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(327): Store=189a4a1fc2d783a754e60fdd039aa718/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:03,654 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,655 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:03,655 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 189a4a1fc2d783a754e60fdd039aa718 columnFamilyName C 2024-12-02T17:15:03,655 DEBUG [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:03,656 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(327): Store=189a4a1fc2d783a754e60fdd039aa718/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:03,656 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,656 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,657 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,657 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,658 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,658 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,659 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:15:03,660 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,663 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:03,663 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 189a4a1fc2d783a754e60fdd039aa718; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59442377, jitterRate=-0.11423955857753754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:15:03,663 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:03,665 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 189a4a1fc2d783a754e60fdd039aa718: Running coprocessor pre-open hook at 1733159703649Writing region info on filesystem at 1733159703649Initializing all the Stores at 1733159703649Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159703649Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159703650 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159703650Cleaning up temporary data from old regions at 1733159703658 (+8 ms)Running coprocessor post-open hooks at 1733159703663 (+5 ms)Region opened successfully at 1733159703665 (+2 ms) 2024-12-02T17:15:03,666 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., pid=36, masterSystemTime=1733159703644 2024-12-02T17:15:03,668 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,668 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:03,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:03,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:15:03,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T17:15:03,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 in 180 msec 2024-12-02T17:15:03,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-02T17:15:03,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, ASSIGN in 337 msec 2024-12-02T17:15:03,677 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:15:03,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159703677"}]},"ts":"1733159703677"} 2024-12-02T17:15:03,679 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T17:15:03,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:15:03,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-12-02T17:15:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T17:15:04,647 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:04,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c92c39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:04,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:04,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:04,650 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11432ffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:04,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:04,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7e2b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:04,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:04,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:04,663 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:15:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:15:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@557415e3 2024-12-02T17:15:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:15:04,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:15:04,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:15:04,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:15:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:04,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741974_1150 (size=1097) 2024-12-02T17:15:05,100 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T17:15:05,100 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T17:15:05,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:15:05,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, REOPEN/MOVE}] 2024-12-02T17:15:05,114 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, REOPEN/MOVE 2024-12-02T17:15:05,115 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, REOPEN/MOVE because future has completed 2024-12-02T17:15:05,117 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:15:05,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:15:05,270 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,270 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:15:05,270 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 189a4a1fc2d783a754e60fdd039aa718, disabling compactions & flushes 2024-12-02T17:15:05,270 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,270 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,270 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. after waiting 0 ms 2024-12-02T17:15:05,270 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,275 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:15:05,275 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,276 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 189a4a1fc2d783a754e60fdd039aa718: Waiting for close lock at 1733159705270Running coprocessor pre-close hooks at 1733159705270Disabling compacts and flushes for region at 1733159705270Disabling writes for close at 1733159705270Writing region close event to WAL at 1733159705271 (+1 ms)Running coprocessor post-close hooks at 1733159705275 (+4 ms)Closed at 1733159705275 2024-12-02T17:15:05,276 WARN [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 189a4a1fc2d783a754e60fdd039aa718 to self. 2024-12-02T17:15:05,278 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,278 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=CLOSED 2024-12-02T17:15:05,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:15:05,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-02T17:15:05,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 in 164 msec 2024-12-02T17:15:05,283 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, REOPEN/MOVE; state=CLOSED, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=true 2024-12-02T17:15:05,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, REOPEN/MOVE because future has completed 2024-12-02T17:15:05,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:15:05,599 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,599 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:15:05,600 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,600 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:05,600 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,600 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,603 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,604 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:05,610 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 189a4a1fc2d783a754e60fdd039aa718 columnFamilyName A 2024-12-02T17:15:05,612 DEBUG [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:05,612 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(327): Store=189a4a1fc2d783a754e60fdd039aa718/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:05,613 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,613 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:05,613 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 189a4a1fc2d783a754e60fdd039aa718 columnFamilyName B 2024-12-02T17:15:05,613 DEBUG [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:05,614 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(327): Store=189a4a1fc2d783a754e60fdd039aa718/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:05,614 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,615 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:05,615 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 189a4a1fc2d783a754e60fdd039aa718 columnFamilyName C 2024-12-02T17:15:05,615 DEBUG [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:05,616 INFO [StoreOpener-189a4a1fc2d783a754e60fdd039aa718-1 {}] regionserver.HStore(327): Store=189a4a1fc2d783a754e60fdd039aa718/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:05,616 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,616 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,617 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,618 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,619 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,619 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,619 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:15:05,620 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,621 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 189a4a1fc2d783a754e60fdd039aa718; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61577765, jitterRate=-0.08241979777812958}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:15:05,621 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,623 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 189a4a1fc2d783a754e60fdd039aa718: Running coprocessor pre-open hook at 1733159705600Writing region info on filesystem at 1733159705600Initializing all the Stores at 1733159705601 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159705601Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159705602 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159705602Cleaning up temporary data from old regions at 1733159705619 (+17 ms)Running coprocessor post-open hooks at 1733159705621 (+2 ms)Region opened successfully at 1733159705623 (+2 ms) 2024-12-02T17:15:05,624 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., pid=41, masterSystemTime=1733159705595 2024-12-02T17:15:05,627 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,627 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,627 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=OPEN, openSeqNum=5, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:15:05,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-02T17:15:05,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 in 193 msec 2024-12-02T17:15:05,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-02T17:15:05,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, REOPEN/MOVE in 519 msec 2024-12-02T17:15:05,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-02T17:15:05,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-12-02T17:15:05,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 965 msec 2024-12-02T17:15:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-02T17:15:05,645 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:05,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,648 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60580cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5b1922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,653 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2b9f49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,657 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@766f6d0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,659 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b6f64e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a4a19c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e4b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0a01e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7731fd86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ff1e65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b32c08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539855c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af31bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2874ba29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,681 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6a8c9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,682 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e4c32e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139bfa79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:05,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:05,685 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e91f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:05,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:05,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:05,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ecc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:05,688 DEBUG [Thread-748 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,688 DEBUG [Thread-744 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,689 DEBUG [Thread-737 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,692 DEBUG [Thread-739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:05,694 DEBUG [Thread-735 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-02T17:15:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:05,696 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,698 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:05,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,702 DEBUG [Thread-733 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,709 DEBUG [Thread-746 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,709 DEBUG [Thread-731 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,710 DEBUG [Thread-742 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:05,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:05,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:15:05,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:05,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:05,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733159705779, max timestamp=1733159705804Num uniques 859; 2024-12-02T17:15:05,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733159705779, max timestamp=1733159705804Num uniques 859; 2024-12-02T17:15:05,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T17:15:05,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733159705779, max timestamp=1733159705804Num uniques 859; 2024-12-02T17:15:05,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T17:15:05,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-02T17:15:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:05,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021e2ae433461a4e159ac4682bc5c47a9a_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159705742/Put/seqid=0 2024-12-02T17:15:05,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:05,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:05,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:05,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159765855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159765855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159765856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159765861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159765868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741975_1151 (size=12154) 2024-12-02T17:15:05,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159765987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159765988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159765988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159765989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:05,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159765990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:05,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159766197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159766197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159766208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159766208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159766208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:06,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021e2ae433461a4e159ac4682bc5c47a9a_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021e2ae433461a4e159ac4682bc5c47a9a_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:06,299 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9452375f9ee744ef9dc3d11f204dc149, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:06,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9452375f9ee744ef9dc3d11f204dc149 is 175, key is test_row_0/A:col10/1733159705742/Put/seqid=0 2024-12-02T17:15:06,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:06,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741976_1152 (size=30955) 2024-12-02T17:15:06,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9452375f9ee744ef9dc3d11f204dc149 2024-12-02T17:15:06,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/86df964b6d244716b211c00324fa9e21 is 50, key is test_row_0/B:col10/1733159705742/Put/seqid=0 2024-12-02T17:15:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741977_1153 (size=12001) 2024-12-02T17:15:06,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/86df964b6d244716b211c00324fa9e21 2024-12-02T17:15:06,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fb7115cc6b434dc18fe73fd67d3cbc2d is 50, key is test_row_0/C:col10/1733159705742/Put/seqid=0 2024-12-02T17:15:06,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159766518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159766519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741978_1154 (size=12001) 2024-12-02T17:15:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159766527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159766527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:06,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159766537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:06,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:06,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:06,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:06,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fb7115cc6b434dc18fe73fd67d3cbc2d 2024-12-02T17:15:06,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9452375f9ee744ef9dc3d11f204dc149 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149 2024-12-02T17:15:06,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T17:15:06,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/86df964b6d244716b211c00324fa9e21 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/86df964b6d244716b211c00324fa9e21 2024-12-02T17:15:06,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:06,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:06,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:06,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/86df964b6d244716b211c00324fa9e21, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T17:15:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fb7115cc6b434dc18fe73fd67d3cbc2d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb7115cc6b434dc18fe73fd67d3cbc2d 2024-12-02T17:15:06,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb7115cc6b434dc18fe73fd67d3cbc2d, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T17:15:06,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 189a4a1fc2d783a754e60fdd039aa718 in 1250ms, sequenceid=15, compaction requested=false 2024-12-02T17:15:06,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:07,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-12-02T17:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:07,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:07,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e750dff39bd749b4ad7cf71a248f5ce8_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159705853/Put/seqid=0 2024-12-02T17:15:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159767062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159767068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159767069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159767070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159767082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741979_1155 (size=14594) 2024-12-02T17:15:07,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:07,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e750dff39bd749b4ad7cf71a248f5ce8_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e750dff39bd749b4ad7cf71a248f5ce8_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:07,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/1fb715d07dab4a64a0200a609ebbf873, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:07,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/1fb715d07dab4a64a0200a609ebbf873 is 175, key is test_row_0/A:col10/1733159705853/Put/seqid=0 2024-12-02T17:15:07,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741980_1156 (size=39549) 2024-12-02T17:15:07,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159767197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159767208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159767208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159767208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159767208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,340 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:15:07,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159767408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159767427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159767427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159767428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159767429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/1fb715d07dab4a64a0200a609ebbf873 2024-12-02T17:15:07,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/1181c8f9c22747c0a0c313040fd08d06 is 50, key is test_row_0/B:col10/1733159705853/Put/seqid=0 2024-12-02T17:15:07,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741981_1157 (size=12001) 2024-12-02T17:15:07,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/1181c8f9c22747c0a0c313040fd08d06 2024-12-02T17:15:07,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/39c13bfe32094b45aa669fd1605052d7 is 50, key is test_row_0/C:col10/1733159705853/Put/seqid=0 2024-12-02T17:15:07,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741982_1158 (size=12001) 2024-12-02T17:15:07,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159767718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159767757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159767758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159767758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159767758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:07,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:07,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:08,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/39c13bfe32094b45aa669fd1605052d7 2024-12-02T17:15:08,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/1fb715d07dab4a64a0200a609ebbf873 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873 2024-12-02T17:15:08,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:08,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:08,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:08,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873, entries=200, sequenceid=48, filesize=38.6 K 2024-12-02T17:15:08,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/1181c8f9c22747c0a0c313040fd08d06 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/1181c8f9c22747c0a0c313040fd08d06 2024-12-02T17:15:08,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/1181c8f9c22747c0a0c313040fd08d06, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T17:15:08,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/39c13bfe32094b45aa669fd1605052d7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/39c13bfe32094b45aa669fd1605052d7 2024-12-02T17:15:08,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/39c13bfe32094b45aa669fd1605052d7, entries=150, sequenceid=48, filesize=11.7 K 2024-12-02T17:15:08,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=53.67 KB/54960 for 189a4a1fc2d783a754e60fdd039aa718 in 1062ms, sequenceid=48, compaction requested=false 2024-12-02T17:15:08,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:08,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-02T17:15:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,230 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:08,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026f7ee8a58b5b4d13bc88abcf913c2ce2_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159707068/Put/seqid=0 2024-12-02T17:15:08,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741983_1159 (size=12154) 2024-12-02T17:15:08,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:08,302 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026f7ee8a58b5b4d13bc88abcf913c2ce2_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026f7ee8a58b5b4d13bc88abcf913c2ce2_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:08,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7613b3a269f14117a524b4af4028b603, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:08,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7613b3a269f14117a524b4af4028b603 is 175, key is test_row_0/A:col10/1733159707068/Put/seqid=0 2024-12-02T17:15:08,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159768313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159768314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159708296, max timestamp=1733159708324Num uniques 859; 2024-12-02T17:15:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159708296, max timestamp=1733159708324Num uniques 859; 2024-12-02T17:15:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:15:08,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:15:08,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733159708296, max timestamp=1733159708324Num uniques 859; 2024-12-02T17:15:08,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-02T17:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741984_1160 (size=30955) 2024-12-02T17:15:08,351 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=59, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7613b3a269f14117a524b4af4028b603 2024-12-02T17:15:08,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/a0f2d2eb540d471f85a1c83a73085925 is 50, key is test_row_0/B:col10/1733159707068/Put/seqid=0 2024-12-02T17:15:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159768383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159768396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159768400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741985_1161 (size=12001) 2024-12-02T17:15:08,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/a0f2d2eb540d471f85a1c83a73085925 2024-12-02T17:15:08,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/5683073b21324034b3eb40110102313b is 50, key is test_row_0/C:col10/1733159707068/Put/seqid=0 2024-12-02T17:15:08,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159768438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159768445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741986_1162 (size=12001) 2024-12-02T17:15:08,471 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/5683073b21324034b3eb40110102313b 2024-12-02T17:15:08,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7613b3a269f14117a524b4af4028b603 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603 2024-12-02T17:15:08,498 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603, entries=150, sequenceid=59, filesize=30.2 K 2024-12-02T17:15:08,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/a0f2d2eb540d471f85a1c83a73085925 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/a0f2d2eb540d471f85a1c83a73085925 2024-12-02T17:15:08,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/a0f2d2eb540d471f85a1c83a73085925, entries=150, sequenceid=59, filesize=11.7 K 2024-12-02T17:15:08,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/5683073b21324034b3eb40110102313b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/5683073b21324034b3eb40110102313b 2024-12-02T17:15:08,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159768507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/5683073b21324034b3eb40110102313b, entries=150, sequenceid=59, filesize=11.7 K 2024-12-02T17:15:08,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for 189a4a1fc2d783a754e60fdd039aa718 in 289ms, sequenceid=59, compaction requested=true 2024-12-02T17:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-02T17:15:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-02T17:15:08,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-02T17:15:08,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8210 sec 2024-12-02T17:15:08,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.8320 sec 2024-12-02T17:15:08,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-02T17:15:08,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:08,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:08,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:08,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:08,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202df4f3c93ea954eae90f2c1af204e7b1d_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:08,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159768563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159768565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741987_1163 (size=14594) 2024-12-02T17:15:08,589 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:08,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202df4f3c93ea954eae90f2c1af204e7b1d_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202df4f3c93ea954eae90f2c1af204e7b1d_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:08,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7e6e5064c1cb477abff8e37e7576374b, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:08,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7e6e5064c1cb477abff8e37e7576374b is 175, key is test_row_0/A:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:08,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741988_1164 (size=39549) 2024-12-02T17:15:08,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7e6e5064c1cb477abff8e37e7576374b 2024-12-02T17:15:08,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/d1a269fc88d64098a1d441c34777f781 is 50, key is test_row_0/B:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159768647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159768659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741989_1165 (size=12001) 2024-12-02T17:15:08,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/d1a269fc88d64098a1d441c34777f781 2024-12-02T17:15:08,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159768687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159768687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fc4177db699c4defb6402a69110feefa is 50, key is test_row_0/C:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159768717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:08,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741990_1166 (size=12001) 2024-12-02T17:15:08,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fc4177db699c4defb6402a69110feefa 2024-12-02T17:15:08,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7e6e5064c1cb477abff8e37e7576374b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b 2024-12-02T17:15:08,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b, entries=200, sequenceid=95, filesize=38.6 K 2024-12-02T17:15:08,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/d1a269fc88d64098a1d441c34777f781 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d1a269fc88d64098a1d441c34777f781 2024-12-02T17:15:08,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d1a269fc88d64098a1d441c34777f781, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T17:15:08,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fc4177db699c4defb6402a69110feefa as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fc4177db699c4defb6402a69110feefa 2024-12-02T17:15:08,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fc4177db699c4defb6402a69110feefa, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T17:15:08,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 189a4a1fc2d783a754e60fdd039aa718 in 232ms, sequenceid=95, compaction requested=true 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:15:08,762 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:08,764 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:08,797 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:08,797 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:08,797 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,797 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb7115cc6b434dc18fe73fd67d3cbc2d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/39c13bfe32094b45aa669fd1605052d7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/5683073b21324034b3eb40110102313b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fc4177db699c4defb6402a69110feefa] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=46.9 K 2024-12-02T17:15:08,798 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141008 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:08,799 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:08,799 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,799 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=137.7 K 2024-12-02T17:15:08,799 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:08,799 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b] 2024-12-02T17:15:08,800 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7115cc6b434dc18fe73fd67d3cbc2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733159705742 2024-12-02T17:15:08,800 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9452375f9ee744ef9dc3d11f204dc149, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733159705742 2024-12-02T17:15:08,801 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c13bfe32094b45aa669fd1605052d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733159705853 2024-12-02T17:15:08,801 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb715d07dab4a64a0200a609ebbf873, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733159705853 2024-12-02T17:15:08,802 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5683073b21324034b3eb40110102313b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733159707053 2024-12-02T17:15:08,802 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7613b3a269f14117a524b4af4028b603, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733159707053 2024-12-02T17:15:08,803 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4177db699c4defb6402a69110feefa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159708362 2024-12-02T17:15:08,803 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e6e5064c1cb477abff8e37e7576374b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159708362 2024-12-02T17:15:08,823 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:08,840 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:08,841 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/cab394faa0d047b49ba7bdf7abe4db6b is 50, key is test_row_0/C:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:08,894 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202ed8a0d353d39408fafcfb0c1ca13d739_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:08,911 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202ed8a0d353d39408fafcfb0c1ca13d739_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:08,912 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202ed8a0d353d39408fafcfb0c1ca13d739_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:08,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:08,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:08,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741991_1167 (size=12139) 2024-12-02T17:15:08,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741992_1168 (size=4469) 2024-12-02T17:15:09,000 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#147 average throughput is 0.14 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:09,001 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/4c9379bfc4784d6d89d19a94e6dabd78 is 175, key is test_row_0/A:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:09,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159708963, max timestamp=1733159708997Num uniques 859; 2024-12-02T17:15:09,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:09,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159708963, max timestamp=1733159708997Num uniques 859; 2024-12-02T17:15:09,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:09,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159708963, max timestamp=1733159708997Num uniques 859; 2024-12-02T17:15:09,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:09,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f2395a1fdad64db1a62b6517aca08d34_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159708908/Put/seqid=0 2024-12-02T17:15:09,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741993_1169 (size=31093) 2024-12-02T17:15:09,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159769059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,098 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/4c9379bfc4784d6d89d19a94e6dabd78 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4c9379bfc4784d6d89d19a94e6dabd78 2024-12-02T17:15:09,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159769076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741994_1170 (size=14594) 2024-12-02T17:15:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159769083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159769087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159769080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,107 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into 4c9379bfc4784d6d89d19a94e6dabd78(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:09,107 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:09,107 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=12, startTime=1733159708761; duration=0sec 2024-12-02T17:15:09,108 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:09,108 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:09,108 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:09,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f2395a1fdad64db1a62b6517aca08d34_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f2395a1fdad64db1a62b6517aca08d34_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:09,112 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/42bdbced56fc43c3a80374ea53144e69, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:09,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/42bdbced56fc43c3a80374ea53144e69 is 175, key is test_row_0/A:col10/1733159708908/Put/seqid=0 2024-12-02T17:15:09,113 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:09,113 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:09,113 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:09,113 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/86df964b6d244716b211c00324fa9e21, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/1181c8f9c22747c0a0c313040fd08d06, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/a0f2d2eb540d471f85a1c83a73085925, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d1a269fc88d64098a1d441c34777f781] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=46.9 K 2024-12-02T17:15:09,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86df964b6d244716b211c00324fa9e21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733159705742 2024-12-02T17:15:09,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1181c8f9c22747c0a0c313040fd08d06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733159705853 2024-12-02T17:15:09,115 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f2d2eb540d471f85a1c83a73085925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733159707053 2024-12-02T17:15:09,115 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1a269fc88d64098a1d441c34777f781, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159708362 2024-12-02T17:15:09,169 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#150 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:09,169 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/af7430839480451b979a891caa6c66e4 is 50, key is test_row_0/B:col10/1733159708394/Put/seqid=0 2024-12-02T17:15:09,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741995_1171 (size=39549) 2024-12-02T17:15:09,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159769207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/42bdbced56fc43c3a80374ea53144e69 2024-12-02T17:15:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159769207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159769207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159769207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159769208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741996_1172 (size=12139) 2024-12-02T17:15:09,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/9dabea3d66254daaa0dea7ef3960624b is 50, key is test_row_0/B:col10/1733159708908/Put/seqid=0 2024-12-02T17:15:09,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741997_1173 (size=12001) 2024-12-02T17:15:09,379 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/cab394faa0d047b49ba7bdf7abe4db6b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/cab394faa0d047b49ba7bdf7abe4db6b 2024-12-02T17:15:09,406 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into cab394faa0d047b49ba7bdf7abe4db6b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:09,406 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:09,406 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=12, startTime=1733159708761; duration=0sec 2024-12-02T17:15:09,406 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:09,406 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:09,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159769425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159769425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159769426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159769427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159769427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,655 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/af7430839480451b979a891caa6c66e4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/af7430839480451b979a891caa6c66e4 2024-12-02T17:15:09,668 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into af7430839480451b979a891caa6c66e4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:09,668 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:09,669 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=12, startTime=1733159708761; duration=0sec 2024-12-02T17:15:09,669 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:09,669 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:09,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/9dabea3d66254daaa0dea7ef3960624b 2024-12-02T17:15:09,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/f690d1dc873a437a9bd1a1cc49f6efd0 is 50, key is test_row_0/C:col10/1733159708908/Put/seqid=0 2024-12-02T17:15:09,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159769747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159769747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159769747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159769747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159769747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:09,785 INFO [master/3b3375d1a925:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T17:15:09,785 INFO [master/3b3375d1a925:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T17:15:09,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741998_1174 (size=12001) 2024-12-02T17:15:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T17:15:09,867 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:09,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-02T17:15:09,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:09,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:09,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:10,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/f690d1dc873a437a9bd1a1cc49f6efd0 2024-12-02T17:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/42bdbced56fc43c3a80374ea53144e69 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69 2024-12-02T17:15:10,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69, entries=200, sequenceid=105, filesize=38.6 K 2024-12-02T17:15:10,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/9dabea3d66254daaa0dea7ef3960624b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/9dabea3d66254daaa0dea7ef3960624b 2024-12-02T17:15:10,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/9dabea3d66254daaa0dea7ef3960624b, entries=150, sequenceid=105, filesize=11.7 K 2024-12-02T17:15:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/f690d1dc873a437a9bd1a1cc49f6efd0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f690d1dc873a437a9bd1a1cc49f6efd0 2024-12-02T17:15:10,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f690d1dc873a437a9bd1a1cc49f6efd0, entries=150, sequenceid=105, filesize=11.7 K 2024-12-02T17:15:10,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 189a4a1fc2d783a754e60fdd039aa718 in 1320ms, sequenceid=105, compaction requested=false 2024-12-02T17:15:10,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:10,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-02T17:15:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:10,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159770277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159770278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159770278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159770282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159770284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e00e8cfef3a84745af916edc50d0a87e_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:10,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741999_1175 (size=12304) 2024-12-02T17:15:10,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159770386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159770386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159770397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159770397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159770397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:10,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159770596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159770607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159770621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159770623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159770621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,732 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:10,737 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e00e8cfef3a84745af916edc50d0a87e_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e00e8cfef3a84745af916edc50d0a87e_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:10,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7cc9c7d20ebe4a3fab57e4619eb1b934, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:10,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7cc9c7d20ebe4a3fab57e4619eb1b934 is 175, key is test_row_0/A:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:10,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742000_1176 (size=31105) 2024-12-02T17:15:10,779 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7cc9c7d20ebe4a3fab57e4619eb1b934 2024-12-02T17:15:10,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/64243daf93324de1a9694a0b0eeddcd3 is 50, key is test_row_0/B:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:10,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742001_1177 (size=12151) 2024-12-02T17:15:10,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/64243daf93324de1a9694a0b0eeddcd3 2024-12-02T17:15:10,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159770907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159770927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159770928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159770937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159770937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:10,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/05c54d6aed534e5a9392e689dbea022b is 50, key is test_row_0/C:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:10,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:10,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742002_1178 (size=12151) 2024-12-02T17:15:11,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:11,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:11,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/05c54d6aed534e5a9392e689dbea022b 2024-12-02T17:15:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159771424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:11,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/7cc9c7d20ebe4a3fab57e4619eb1b934 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934 2024-12-02T17:15:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:11,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:11,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934, entries=150, sequenceid=145, filesize=30.4 K 2024-12-02T17:15:11,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/64243daf93324de1a9694a0b0eeddcd3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/64243daf93324de1a9694a0b0eeddcd3 2024-12-02T17:15:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159771447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:11,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/64243daf93324de1a9694a0b0eeddcd3, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T17:15:11,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/05c54d6aed534e5a9392e689dbea022b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/05c54d6aed534e5a9392e689dbea022b 2024-12-02T17:15:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159771447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159771448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:11,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/05c54d6aed534e5a9392e689dbea022b, entries=150, sequenceid=145, filesize=11.9 K 2024-12-02T17:15:11,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 189a4a1fc2d783a754e60fdd039aa718 in 1191ms, sequenceid=145, compaction requested=true 2024-12-02T17:15:11,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:11,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:11,459 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:11,460 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:11,461 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101747 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,461 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:11,461 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,461 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4c9379bfc4784d6d89d19a94e6dabd78, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=99.4 K 2024-12-02T17:15:11,461 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,461 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4c9379bfc4784d6d89d19a94e6dabd78, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934] 2024-12-02T17:15:11,462 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c9379bfc4784d6d89d19a94e6dabd78, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159708362 2024-12-02T17:15:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,462 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:11,463 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:11,463 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,463 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/af7430839480451b979a891caa6c66e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/9dabea3d66254daaa0dea7ef3960624b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/64243daf93324de1a9694a0b0eeddcd3] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=35.4 K 2024-12-02T17:15:11,463 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42bdbced56fc43c3a80374ea53144e69, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733159708540 2024-12-02T17:15:11,463 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting af7430839480451b979a891caa6c66e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159708362 2024-12-02T17:15:11,464 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc9c7d20ebe4a3fab57e4619eb1b934, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733159709051 2024-12-02T17:15:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,465 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dabea3d66254daaa0dea7ef3960624b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733159708540 2024-12-02T17:15:11,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,466 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 64243daf93324de1a9694a0b0eeddcd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733159709051 2024-12-02T17:15:11,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:11,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:11,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:11,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:11,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:11,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:11,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:11,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,480 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,489 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:11,490 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/31237a7783fa4231a23df83962c8ec40 is 50, key is test_row_0/B:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b52a9df9458f4fb49c894dc400be9a4d_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159711458/Put/seqid=0 2024-12-02T17:15:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,516 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202a93ba0c76ca0435786a97f5848027db5_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:11,520 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202a93ba0c76ca0435786a97f5848027db5_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:11,520 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a93ba0c76ca0435786a97f5848027db5_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742004_1180 (size=14794) 2024-12-02T17:15:11,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742003_1179 (size=12391) 2024-12-02T17:15:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:11,572 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b52a9df9458f4fb49c894dc400be9a4d_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b52a9df9458f4fb49c894dc400be9a4d_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:11,573 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/18c061b104574dc8884253a9726f8ee9, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:11,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/18c061b104574dc8884253a9726f8ee9 is 175, key is test_row_0/A:col10/1733159711458/Put/seqid=0 2024-12-02T17:15:11,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742005_1181 (size=4469) 2024-12-02T17:15:11,600 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#156 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:11,601 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/a4f7a294c91a47829c8ef4ae95e1d459 is 175, key is test_row_0/A:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:11,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742006_1182 (size=39749) 2024-12-02T17:15:11,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/18c061b104574dc8884253a9726f8ee9 2024-12-02T17:15:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742007_1183 (size=31345) 2024-12-02T17:15:11,660 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/a4f7a294c91a47829c8ef4ae95e1d459 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/a4f7a294c91a47829c8ef4ae95e1d459 2024-12-02T17:15:11,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5cd4d6b7c87a492c99946b34f79e0597 is 50, key is test_row_0/B:col10/1733159711458/Put/seqid=0 2024-12-02T17:15:11,668 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into a4f7a294c91a47829c8ef4ae95e1d459(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:11,669 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:11,669 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=13, startTime=1733159711459; duration=0sec 2024-12-02T17:15:11,669 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:11,669 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:11,669 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:11,671 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:11,671 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:11,671 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,671 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/cab394faa0d047b49ba7bdf7abe4db6b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f690d1dc873a437a9bd1a1cc49f6efd0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/05c54d6aed534e5a9392e689dbea022b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=35.4 K 2024-12-02T17:15:11,672 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab394faa0d047b49ba7bdf7abe4db6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159708362 2024-12-02T17:15:11,672 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f690d1dc873a437a9bd1a1cc49f6efd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1733159708540 2024-12-02T17:15:11,673 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c54d6aed534e5a9392e689dbea022b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733159709051 2024-12-02T17:15:11,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159711634, max timestamp=1733159711671Num uniques 859; 2024-12-02T17:15:11,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:11,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159711634, max timestamp=1733159711671Num uniques 859; 2024-12-02T17:15:11,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159711634, max timestamp=1733159711671Num uniques 859; 2024-12-02T17:15:11,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:11,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:11,692 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:11,693 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/33ffea0ce0684eb5906400e8723c3871 is 50, key is test_row_0/C:col10/1733159710263/Put/seqid=0 2024-12-02T17:15:11,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742008_1184 (size=12151) 2024-12-02T17:15:11,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742009_1185 (size=12391) 2024-12-02T17:15:11,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159771795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:11,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:11,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:11,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:11,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:11,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159771908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:11,970 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/31237a7783fa4231a23df83962c8ec40 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/31237a7783fa4231a23df83962c8ec40 2024-12-02T17:15:11,982 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into 31237a7783fa4231a23df83962c8ec40(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:11,982 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:11,982 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=13, startTime=1733159711460; duration=0sec 2024-12-02T17:15:11,982 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:11,982 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:12,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:12,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:12,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5cd4d6b7c87a492c99946b34f79e0597 2024-12-02T17:15:12,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/b7914f2e173c4975866db2314872f16f is 50, key is test_row_0/C:col10/1733159711458/Put/seqid=0 2024-12-02T17:15:12,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159772137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,146 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/33ffea0ce0684eb5906400e8723c3871 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/33ffea0ce0684eb5906400e8723c3871 2024-12-02T17:15:12,152 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into 33ffea0ce0684eb5906400e8723c3871(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:12,152 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:12,152 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=13, startTime=1733159711460; duration=0sec 2024-12-02T17:15:12,152 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:12,152 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:12,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742010_1186 (size=12151) 2024-12-02T17:15:12,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/b7914f2e173c4975866db2314872f16f 2024-12-02T17:15:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/18c061b104574dc8884253a9726f8ee9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9 2024-12-02T17:15:12,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9, entries=200, sequenceid=155, filesize=38.8 K 2024-12-02T17:15:12,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5cd4d6b7c87a492c99946b34f79e0597 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5cd4d6b7c87a492c99946b34f79e0597 2024-12-02T17:15:12,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5cd4d6b7c87a492c99946b34f79e0597, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T17:15:12,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/b7914f2e173c4975866db2314872f16f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/b7914f2e173c4975866db2314872f16f 2024-12-02T17:15:12,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/b7914f2e173c4975866db2314872f16f, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T17:15:12,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 189a4a1fc2d783a754e60fdd039aa718 in 731ms, sequenceid=155, compaction requested=false 2024-12-02T17:15:12,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:12,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-02T17:15:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:12,224 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T17:15:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:12,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:12,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:12,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e5dedb6d36cd43638159d7342ab39a40_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159711768/Put/seqid=0 2024-12-02T17:15:12,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742011_1187 (size=12304) 2024-12-02T17:15:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:12,290 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e5dedb6d36cd43638159d7342ab39a40_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e5dedb6d36cd43638159d7342ab39a40_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:12,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/ceeb7d9456484024b2640909d6482815, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:12,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/ceeb7d9456484024b2640909d6482815 is 175, key is test_row_0/A:col10/1733159711768/Put/seqid=0 2024-12-02T17:15:12,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742012_1188 (size=31105) 2024-12-02T17:15:12,332 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/ceeb7d9456484024b2640909d6482815 2024-12-02T17:15:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/bf3a09f24e484978b28bd6fe8b68e0a1 is 50, key is test_row_0/B:col10/1733159711768/Put/seqid=0 2024-12-02T17:15:12,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742013_1189 (size=12151) 2024-12-02T17:15:12,413 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/bf3a09f24e484978b28bd6fe8b68e0a1 2024-12-02T17:15:12,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6e436a26fa12483fb003c921ca530407 is 50, key is test_row_0/C:col10/1733159711768/Put/seqid=0 2024-12-02T17:15:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:12,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742014_1190 (size=12151) 2024-12-02T17:15:12,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6e436a26fa12483fb003c921ca530407 2024-12-02T17:15:12,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/ceeb7d9456484024b2640909d6482815 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815 2024-12-02T17:15:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159772468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,477 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 189a4a1fc2d783a754e60fdd039aa718 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:12,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159772468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159772472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,478 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 189a4a1fc2d783a754e60fdd039aa718 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,479 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815, entries=150, sequenceid=193, filesize=30.4 K 2024-12-02T17:15:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/bf3a09f24e484978b28bd6fe8b68e0a1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bf3a09f24e484978b28bd6fe8b68e0a1 2024-12-02T17:15:12,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159772475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159772477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,486 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bf3a09f24e484978b28bd6fe8b68e0a1, entries=150, sequenceid=193, filesize=11.9 K 2024-12-02T17:15:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6e436a26fa12483fb003c921ca530407 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6e436a26fa12483fb003c921ca530407 2024-12-02T17:15:12,498 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6e436a26fa12483fb003c921ca530407, entries=150, sequenceid=193, filesize=11.9 K 2024-12-02T17:15:12,500 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 189a4a1fc2d783a754e60fdd039aa718 in 276ms, sequenceid=193, compaction requested=true 2024-12-02T17:15:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:12,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-02T17:15:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-02T17:15:12,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-02T17:15:12,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-12-02T17:15:12,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.6370 sec 2024-12-02T17:15:12,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:12,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:12,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ebe4682eaf44331aa537f43099006f6_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:12,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159712632, max timestamp=1733159712661Num uniques 859; 2024-12-02T17:15:12,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:12,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159712632, max timestamp=1733159712661Num uniques 859; 2024-12-02T17:15:12,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159712632, max timestamp=1733159712661Num uniques 859; 2024-12-02T17:15:12,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:12,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:15:12,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742015_1191 (size=14794) 2024-12-02T17:15:12,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:12,679 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025ebe4682eaf44331aa537f43099006f6_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ebe4682eaf44331aa537f43099006f6_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:12,681 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/5e0661f86e1749a68f965bbc8bdbd6bd, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:12,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/5e0661f86e1749a68f965bbc8bdbd6bd is 175, key is test_row_0/A:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:12,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742016_1192 (size=39749) 2024-12-02T17:15:12,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159772705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159772716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159772718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159772827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159772837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159772838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159773047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159773057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159773059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/5e0661f86e1749a68f965bbc8bdbd6bd 2024-12-02T17:15:13,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/520603e3a4a7481fa664be050b2bfa76 is 50, key is test_row_0/B:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:13,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742017_1193 (size=12151) 2024-12-02T17:15:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159773367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159773367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159773377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/520603e3a4a7481fa664be050b2bfa76 2024-12-02T17:15:13,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/a4f0dcb691114c239bff24fe19d4dc4b is 50, key is test_row_0/C:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742018_1194 (size=12151) 2024-12-02T17:15:13,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/a4f0dcb691114c239bff24fe19d4dc4b 2024-12-02T17:15:13,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/5e0661f86e1749a68f965bbc8bdbd6bd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd 2024-12-02T17:15:13,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd, entries=200, sequenceid=204, filesize=38.8 K 2024-12-02T17:15:13,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/520603e3a4a7481fa664be050b2bfa76 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/520603e3a4a7481fa664be050b2bfa76 2024-12-02T17:15:13,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/520603e3a4a7481fa664be050b2bfa76, entries=150, sequenceid=204, filesize=11.9 K 2024-12-02T17:15:13,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/a4f0dcb691114c239bff24fe19d4dc4b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/a4f0dcb691114c239bff24fe19d4dc4b 2024-12-02T17:15:13,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/a4f0dcb691114c239bff24fe19d4dc4b, entries=150, sequenceid=204, filesize=11.9 K 2024-12-02T17:15:13,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 189a4a1fc2d783a754e60fdd039aa718 in 1167ms, sequenceid=204, compaction requested=true 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:13,759 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:13,759 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:13,761 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:13,761 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:13,761 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:13,761 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/a4f7a294c91a47829c8ef4ae95e1d459, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=138.6 K 2024-12-02T17:15:13,761 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:13,761 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/a4f7a294c91a47829c8ef4ae95e1d459, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd] 2024-12-02T17:15:13,762 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:13,762 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:13,762 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4f7a294c91a47829c8ef4ae95e1d459, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733159709051 2024-12-02T17:15:13,762 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:13,762 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/31237a7783fa4231a23df83962c8ec40, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5cd4d6b7c87a492c99946b34f79e0597, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bf3a09f24e484978b28bd6fe8b68e0a1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/520603e3a4a7481fa664be050b2bfa76] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=47.7 K 2024-12-02T17:15:13,769 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 31237a7783fa4231a23df83962c8ec40, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733159709051 2024-12-02T17:15:13,769 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c061b104574dc8884253a9726f8ee9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733159710276 2024-12-02T17:15:13,769 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceeb7d9456484024b2640909d6482815, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733159711741 2024-12-02T17:15:13,770 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd4d6b7c87a492c99946b34f79e0597, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733159710283 2024-12-02T17:15:13,770 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e0661f86e1749a68f965bbc8bdbd6bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733159712458 2024-12-02T17:15:13,771 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting bf3a09f24e484978b28bd6fe8b68e0a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733159711741 2024-12-02T17:15:13,771 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 520603e3a4a7481fa664be050b2bfa76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733159712458 2024-12-02T17:15:13,793 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:13,820 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412024d03ec1828464b9ba374e61370ec1715_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:13,820 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#169 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:13,821 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/abc8adb61b884fa79efae2934a7eb033 is 50, key is test_row_0/B:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:13,823 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412024d03ec1828464b9ba374e61370ec1715_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:13,824 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024d03ec1828464b9ba374e61370ec1715_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:13,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742019_1195 (size=12527) 2024-12-02T17:15:13,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742020_1196 (size=4469) 2024-12-02T17:15:13,880 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#168 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:13,881 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/5752298f15504b93b23db3d53833ee2b is 175, key is test_row_0/A:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:13,889 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/abc8adb61b884fa79efae2934a7eb033 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/abc8adb61b884fa79efae2934a7eb033 2024-12-02T17:15:13,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742021_1197 (size=31481) 2024-12-02T17:15:13,898 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into abc8adb61b884fa79efae2934a7eb033(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:13,898 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:13,898 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=12, startTime=1733159713759; duration=0sec 2024-12-02T17:15:13,898 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:13,898 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:13,898 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:13,900 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:13,900 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:13,901 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:13,901 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/33ffea0ce0684eb5906400e8723c3871, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/b7914f2e173c4975866db2314872f16f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6e436a26fa12483fb003c921ca530407, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/a4f0dcb691114c239bff24fe19d4dc4b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=47.7 K 2024-12-02T17:15:13,902 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ffea0ce0684eb5906400e8723c3871, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733159709051 2024-12-02T17:15:13,902 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/5752298f15504b93b23db3d53833ee2b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5752298f15504b93b23db3d53833ee2b 2024-12-02T17:15:13,903 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b7914f2e173c4975866db2314872f16f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733159710283 2024-12-02T17:15:13,904 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e436a26fa12483fb003c921ca530407, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733159711741 2024-12-02T17:15:13,904 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f0dcb691114c239bff24fe19d4dc4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733159712458 2024-12-02T17:15:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:13,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-02T17:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:13,913 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into 5752298f15504b93b23db3d53833ee2b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:13,913 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:13,913 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=12, startTime=1733159713759; duration=0sec 2024-12-02T17:15:13,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:13,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:13,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159773933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159773936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c7026de2c6414630a649ed519078364e_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159712699/Put/seqid=0 2024-12-02T17:15:13,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159773938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:13,974 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:13,975 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/609f0b63d4c0443e80a395e5e1993f76 is 50, key is test_row_0/C:col10/1733159712458/Put/seqid=0 2024-12-02T17:15:14,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742022_1198 (size=14794) 2024-12-02T17:15:14,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:14,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c7026de2c6414630a649ed519078364e_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7026de2c6414630a649ed519078364e_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:14,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/02e9b2c53ef743aa8288fea49b05878b, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:14,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/02e9b2c53ef743aa8288fea49b05878b is 175, key is test_row_0/A:col10/1733159712699/Put/seqid=0 2024-12-02T17:15:14,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742023_1199 (size=12527) 2024-12-02T17:15:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-02T17:15:14,028 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:14,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-02T17:15:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T17:15:14,031 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:14,032 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:14,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:14,038 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/609f0b63d4c0443e80a395e5e1993f76 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/609f0b63d4c0443e80a395e5e1993f76 2024-12-02T17:15:14,046 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into 609f0b63d4c0443e80a395e5e1993f76(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:14,046 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:14,046 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=12, startTime=1733159713759; duration=0sec 2024-12-02T17:15:14,046 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:14,046 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159774047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159774057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159774057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742024_1200 (size=39749) 2024-12-02T17:15:14,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/02e9b2c53ef743aa8288fea49b05878b 2024-12-02T17:15:14,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/3c8c1af0b280418e9d1231b6c0c0dcb5 is 50, key is test_row_0/B:col10/1733159712699/Put/seqid=0 2024-12-02T17:15:14,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742025_1201 (size=12151) 2024-12-02T17:15:14,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/3c8c1af0b280418e9d1231b6c0c0dcb5 2024-12-02T17:15:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T17:15:14,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ac0db473f061497f9d4a16f18400cbf7 is 50, key is test_row_0/C:col10/1733159712699/Put/seqid=0 2024-12-02T17:15:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742026_1202 (size=12151) 2024-12-02T17:15:14,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T17:15:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159774257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159774267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159774276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T17:15:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T17:15:14,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T17:15:14,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:14,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159774497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159774497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,500 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 189a4a1fc2d783a754e60fdd039aa718 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,501 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 189a4a1fc2d783a754e60fdd039aa718 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159774567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ac0db473f061497f9d4a16f18400cbf7 2024-12-02T17:15:14,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/02e9b2c53ef743aa8288fea49b05878b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b 2024-12-02T17:15:14,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159774577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b, entries=200, sequenceid=240, filesize=38.8 K 2024-12-02T17:15:14,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/3c8c1af0b280418e9d1231b6c0c0dcb5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/3c8c1af0b280418e9d1231b6c0c0dcb5 2024-12-02T17:15:14,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/3c8c1af0b280418e9d1231b6c0c0dcb5, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T17:15:14,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ac0db473f061497f9d4a16f18400cbf7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ac0db473f061497f9d4a16f18400cbf7 2024-12-02T17:15:14,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ac0db473f061497f9d4a16f18400cbf7, entries=150, sequenceid=240, filesize=11.9 K 2024-12-02T17:15:14,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159774599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:14,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:14,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 189a4a1fc2d783a754e60fdd039aa718 in 705ms, sequenceid=240, compaction requested=false 2024-12-02T17:15:14,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:14,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-02T17:15:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:14,649 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T17:15:14,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fd8ed05c87804c9d9a6b8aced48824b3_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159713916/Put/seqid=0 2024-12-02T17:15:14,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742027_1203 (size=12304) 2024-12-02T17:15:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:14,706 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fd8ed05c87804c9d9a6b8aced48824b3_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fd8ed05c87804c9d9a6b8aced48824b3_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:14,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/8270f99af3d3458690b1709262c762b2, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:14,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/8270f99af3d3458690b1709262c762b2 is 175, key is test_row_0/A:col10/1733159713916/Put/seqid=0 2024-12-02T17:15:14,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742028_1204 (size=31105) 2024-12-02T17:15:14,742 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/8270f99af3d3458690b1709262c762b2 2024-12-02T17:15:14,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/cbba9e457e3b43e9ac22b577c268b243 is 50, key is test_row_0/B:col10/1733159713916/Put/seqid=0 2024-12-02T17:15:14,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742029_1205 (size=12151) 2024-12-02T17:15:14,795 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/cbba9e457e3b43e9ac22b577c268b243 2024-12-02T17:15:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/be1e6804f9c54416806159d83275b87e is 50, key is test_row_0/C:col10/1733159713916/Put/seqid=0 2024-12-02T17:15:14,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742030_1206 (size=12151) 2024-12-02T17:15:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:15,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:15,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159715123, max timestamp=1733159715137Num uniques 859; 2024-12-02T17:15:15,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159715123, max timestamp=1733159715137Num uniques 859; 2024-12-02T17:15:15,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159715123, max timestamp=1733159715137Num uniques 859; 2024-12-02T17:15:15,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:15,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:15,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T17:15:15,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159775180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159775185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159775186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,231 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/be1e6804f9c54416806159d83275b87e 2024-12-02T17:15:15,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/8270f99af3d3458690b1709262c762b2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2 2024-12-02T17:15:15,245 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2, entries=150, sequenceid=252, filesize=30.4 K 2024-12-02T17:15:15,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/cbba9e457e3b43e9ac22b577c268b243 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/cbba9e457e3b43e9ac22b577c268b243 2024-12-02T17:15:15,257 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/cbba9e457e3b43e9ac22b577c268b243, entries=150, sequenceid=252, filesize=11.9 K 2024-12-02T17:15:15,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/be1e6804f9c54416806159d83275b87e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/be1e6804f9c54416806159d83275b87e 2024-12-02T17:15:15,266 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/be1e6804f9c54416806159d83275b87e, entries=150, sequenceid=252, filesize=11.9 K 2024-12-02T17:15:15,269 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 189a4a1fc2d783a754e60fdd039aa718 in 621ms, sequenceid=252, compaction requested=true 2024-12-02T17:15:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:15,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-02T17:15:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-02T17:15:15,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-02T17:15:15,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2410 sec 2024-12-02T17:15:15,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2460 sec 2024-12-02T17:15:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:15,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-02T17:15:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:15:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:15:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:15,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:15:15,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159775311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159775315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159775317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120235d51995941f40d3b8af070d58c11d41_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742031_1207 (size=12454) 2024-12-02T17:15:15,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120235d51995941f40d3b8af070d58c11d41_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120235d51995941f40d3b8af070d58c11d41_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:15,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/d00a69d9c99c46928275a609b49f31fe, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:15,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/d00a69d9c99c46928275a609b49f31fe is 175, key is test_row_0/A:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742032_1208 (size=31255) 2024-12-02T17:15:15,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/d00a69d9c99c46928275a609b49f31fe 2024-12-02T17:15:15,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/17a9b6c525aa4b599261c7a4a573d8e1 is 50, key is test_row_0/B:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742033_1209 (size=12301) 2024-12-02T17:15:15,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/17a9b6c525aa4b599261c7a4a573d8e1 2024-12-02T17:15:15,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/15bb1c01f03b4fce975f531438bacb65 is 50, key is test_row_0/C:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159775428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159775428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159775433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742034_1210 (size=12301) 2024-12-02T17:15:15,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/15bb1c01f03b4fce975f531438bacb65 2024-12-02T17:15:15,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/d00a69d9c99c46928275a609b49f31fe as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe 2024-12-02T17:15:15,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe, entries=150, sequenceid=291, filesize=30.5 K 2024-12-02T17:15:15,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/17a9b6c525aa4b599261c7a4a573d8e1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/17a9b6c525aa4b599261c7a4a573d8e1 2024-12-02T17:15:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/17a9b6c525aa4b599261c7a4a573d8e1, entries=150, sequenceid=291, filesize=12.0 K 2024-12-02T17:15:15,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/15bb1c01f03b4fce975f531438bacb65 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/15bb1c01f03b4fce975f531438bacb65 2024-12-02T17:15:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/15bb1c01f03b4fce975f531438bacb65, entries=150, sequenceid=291, filesize=12.0 K 2024-12-02T17:15:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for 189a4a1fc2d783a754e60fdd039aa718 in 204ms, sequenceid=291, compaction requested=true 2024-12-02T17:15:15,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:15,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:15:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:15,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:15:15,506 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,506 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,512 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133590 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:15,512 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:15,512 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:15,512 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:15,512 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:15,512 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:15,512 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5752298f15504b93b23db3d53833ee2b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=130.5 K 2024-12-02T17:15:15,512 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/609f0b63d4c0443e80a395e5e1993f76, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ac0db473f061497f9d4a16f18400cbf7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/be1e6804f9c54416806159d83275b87e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/15bb1c01f03b4fce975f531438bacb65] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.0 K 2024-12-02T17:15:15,512 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:15,512 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5752298f15504b93b23db3d53833ee2b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe] 2024-12-02T17:15:15,512 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 609f0b63d4c0443e80a395e5e1993f76, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733159712458 2024-12-02T17:15:15,513 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5752298f15504b93b23db3d53833ee2b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733159712458 2024-12-02T17:15:15,513 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0db473f061497f9d4a16f18400cbf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733159712699 2024-12-02T17:15:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,514 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02e9b2c53ef743aa8288fea49b05878b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733159712699 2024-12-02T17:15:15,514 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting be1e6804f9c54416806159d83275b87e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733159713916 2024-12-02T17:15:15,514 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8270f99af3d3458690b1709262c762b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733159713916 2024-12-02T17:15:15,514 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d00a69d9c99c46928275a609b49f31fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733159715146 2024-12-02T17:15:15,514 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 15bb1c01f03b4fce975f531438bacb65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733159715146 2024-12-02T17:15:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,535 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#180 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,535 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/e4f5b4ea622347dc852b28dc35d77c49 is 50, key is test_row_0/C:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,539 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742035_1211 (size=12813) 2024-12-02T17:15:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,570 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202fd60c6d6e29945a7ae85d3fd5f2d6b73_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,572 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202fd60c6d6e29945a7ae85d3fd5f2d6b73_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,573 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fd60c6d6e29945a7ae85d3fd5f2d6b73_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,575 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/e4f5b4ea622347dc852b28dc35d77c49 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e4f5b4ea622347dc852b28dc35d77c49 2024-12-02T17:15:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,592 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into e4f5b4ea622347dc852b28dc35d77c49(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:15,592 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:15,592 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=12, startTime=1733159715506; duration=0sec 2024-12-02T17:15:15,592 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:15,592 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:15,592 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,595 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:15,595 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,595 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:15,595 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/abc8adb61b884fa79efae2934a7eb033, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/3c8c1af0b280418e9d1231b6c0c0dcb5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/cbba9e457e3b43e9ac22b577c268b243, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/17a9b6c525aa4b599261c7a4a573d8e1] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.0 K 2024-12-02T17:15:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,597 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting abc8adb61b884fa79efae2934a7eb033, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733159712458 2024-12-02T17:15:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,598 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c8c1af0b280418e9d1231b6c0c0dcb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733159712699 2024-12-02T17:15:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,598 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting cbba9e457e3b43e9ac22b577c268b243, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733159713916 2024-12-02T17:15:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,599 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a9b6c525aa4b599261c7a4a573d8e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733159715146 2024-12-02T17:15:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742036_1212 (size=4469) 2024-12-02T17:15:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,605 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#181 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:15,605 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/ef8f1b7f486e4142b3f65af4a5250ea5 is 175, key is test_row_0/A:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742037_1213 (size=31767) 2024-12-02T17:15:15,619 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:15,619 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/21f0bf081e0147a59c1cec1888636650 is 50, key is test_row_0/B:col10/1733159715146/Put/seqid=0 2024-12-02T17:15:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,629 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/ef8f1b7f486e4142b3f65af4a5250ea5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ef8f1b7f486e4142b3f65af4a5250ea5 2024-12-02T17:15:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742038_1214 (size=12813) 2024-12-02T17:15:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,648 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into ef8f1b7f486e4142b3f65af4a5250ea5(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:15,648 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:15,648 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=12, startTime=1733159715505; duration=0sec 2024-12-02T17:15:15,648 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:15,648 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,660 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/21f0bf081e0147a59c1cec1888636650 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/21f0bf081e0147a59c1cec1888636650 2024-12-02T17:15:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,687 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into 21f0bf081e0147a59c1cec1888636650(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:15,687 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:15,687 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=12, startTime=1733159715506; duration=0sec 2024-12-02T17:15:15,688 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:15,688 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c631a299fb5448238b0097ea4e59bc23_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159715660/Put/seqid=0 2024-12-02T17:15:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742039_1215 (size=17534) 2024-12-02T17:15:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159775744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159775755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159775756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159775869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159775873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159775873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:15,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159776077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159776087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159776096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,127 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:16,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c631a299fb5448238b0097ea4e59bc23_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c631a299fb5448238b0097ea4e59bc23_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:16,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/0347e9c513424454bc8032963c086978, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:16,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/0347e9c513424454bc8032963c086978 is 175, key is test_row_0/A:col10/1733159715660/Put/seqid=0 2024-12-02T17:15:16,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742040_1216 (size=48639) 2024-12-02T17:15:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-02T17:15:16,177 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:16,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-02T17:15:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T17:15:16,180 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:16,181 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:16,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T17:15:16,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T17:15:16,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:16,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:16,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:16,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:16,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:16,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159776387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159776397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159776407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T17:15:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:16,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T17:15:16,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/0347e9c513424454bc8032963c086978 2024-12-02T17:15:16,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/43f513ccc8ce446e8e61ea38bdc1f77c is 50, key is test_row_0/B:col10/1733159715660/Put/seqid=0 2024-12-02T17:15:16,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742041_1217 (size=12301) 2024-12-02T17:15:16,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/43f513ccc8ce446e8e61ea38bdc1f77c 2024-12-02T17:15:16,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/91550c9b6232446a93abe618a83ea0a1 is 50, key is test_row_0/C:col10/1733159715660/Put/seqid=0 2024-12-02T17:15:16,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742042_1218 (size=12301) 2024-12-02T17:15:16,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/91550c9b6232446a93abe618a83ea0a1 2024-12-02T17:15:16,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/0347e9c513424454bc8032963c086978 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978 2024-12-02T17:15:16,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978, entries=250, sequenceid=303, filesize=47.5 K 2024-12-02T17:15:16,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/43f513ccc8ce446e8e61ea38bdc1f77c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/43f513ccc8ce446e8e61ea38bdc1f77c 2024-12-02T17:15:16,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/43f513ccc8ce446e8e61ea38bdc1f77c, entries=150, sequenceid=303, filesize=12.0 K 2024-12-02T17:15:16,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/91550c9b6232446a93abe618a83ea0a1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/91550c9b6232446a93abe618a83ea0a1 2024-12-02T17:15:16,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/91550c9b6232446a93abe618a83ea0a1, entries=150, sequenceid=303, filesize=12.0 K 2024-12-02T17:15:16,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 189a4a1fc2d783a754e60fdd039aa718 in 961ms, sequenceid=303, compaction requested=false 2024-12-02T17:15:16,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:16,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-02T17:15:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:16,640 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-02T17:15:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:16,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:16,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c6de9d65a2ac4719854ef00a8608ab53_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159715753/Put/seqid=0 2024-12-02T17:15:16,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742043_1219 (size=12454) 2024-12-02T17:15:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T17:15:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:16,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:16,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159776920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159776922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159776923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159777037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159777038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159777038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:17,072 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c6de9d65a2ac4719854ef00a8608ab53_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c6de9d65a2ac4719854ef00a8608ab53_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/33045c7876d240989ab95b87a98892e4, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/33045c7876d240989ab95b87a98892e4 is 175, key is test_row_0/A:col10/1733159715753/Put/seqid=0 2024-12-02T17:15:17,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742044_1220 (size=31255) 2024-12-02T17:15:17,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/33045c7876d240989ab95b87a98892e4 2024-12-02T17:15:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/2d0ad9a925c14844b7188b081d906861 is 50, key is test_row_0/B:col10/1733159715753/Put/seqid=0 2024-12-02T17:15:17,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742045_1221 (size=12301) 2024-12-02T17:15:17,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/2d0ad9a925c14844b7188b081d906861 2024-12-02T17:15:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/d07e476967cb438fbe23c6a9f1b9ad0c is 50, key is test_row_0/C:col10/1733159715753/Put/seqid=0 2024-12-02T17:15:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742046_1222 (size=12301) 2024-12-02T17:15:17,231 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/d07e476967cb438fbe23c6a9f1b9ad0c 2024-12-02T17:15:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/33045c7876d240989ab95b87a98892e4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4 2024-12-02T17:15:17,248 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4, entries=150, sequenceid=326, filesize=30.5 K 2024-12-02T17:15:17,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/2d0ad9a925c14844b7188b081d906861 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/2d0ad9a925c14844b7188b081d906861 2024-12-02T17:15:17,255 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/2d0ad9a925c14844b7188b081d906861, entries=150, sequenceid=326, filesize=12.0 K 2024-12-02T17:15:17,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/d07e476967cb438fbe23c6a9f1b9ad0c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/d07e476967cb438fbe23c6a9f1b9ad0c 2024-12-02T17:15:17,263 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/d07e476967cb438fbe23c6a9f1b9ad0c, entries=150, sequenceid=326, filesize=12.0 K 2024-12-02T17:15:17,269 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 189a4a1fc2d783a754e60fdd039aa718 in 629ms, sequenceid=326, compaction requested=true 2024-12-02T17:15:17,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-02T17:15:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-02T17:15:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:17,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:15:17,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-02T17:15:17,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-12-02T17:15:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:17,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0980 sec 2024-12-02T17:15:17,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e454e688ed2448db9c6a246679089beb_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-02T17:15:17,317 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:17,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-02T17:15:17,320 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:17,321 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:17,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:17,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159777327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159777331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159777332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742047_1223 (size=12454) 2024-12-02T17:15:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:17,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159777437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159777447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159777448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:17,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159777647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159777657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159777659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,753 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:17,758 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e454e688ed2448db9c6a246679089beb_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e454e688ed2448db9c6a246679089beb_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:17,759 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/904cfcf4c2474ba9891d36411f3e0cf2, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:17,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/904cfcf4c2474ba9891d36411f3e0cf2 is 175, key is test_row_0/A:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:17,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742048_1224 (size=31255) 2024-12-02T17:15:17,764 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/904cfcf4c2474ba9891d36411f3e0cf2 2024-12-02T17:15:17,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/595331ce8667426c9e36716f2e5e21a5 is 50, key is test_row_0/B:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:17,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742049_1225 (size=12301) 2024-12-02T17:15:17,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:17,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:17,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:17,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:17,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159777957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159777969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159777969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:18,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/595331ce8667426c9e36716f2e5e21a5 2024-12-02T17:15:18,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/3897501f3af549d8aedc91742e4e38f6 is 50, key is test_row_0/C:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:18,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742050_1226 (size=12301) 2024-12-02T17:15:18,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:18,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159778467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159778477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159778477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:18,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59602 deadline: 1733159778527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:18,530 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 189a4a1fc2d783a754e60fdd039aa718 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59574 deadline: 1733159778547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:18,549 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 189a4a1fc2d783a754e60fdd039aa718 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:18,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:18,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:18,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:18,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/3897501f3af549d8aedc91742e4e38f6 2024-12-02T17:15:18,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/904cfcf4c2474ba9891d36411f3e0cf2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2 2024-12-02T17:15:18,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2, entries=150, sequenceid=340, filesize=30.5 K 2024-12-02T17:15:18,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/595331ce8667426c9e36716f2e5e21a5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/595331ce8667426c9e36716f2e5e21a5 2024-12-02T17:15:18,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/595331ce8667426c9e36716f2e5e21a5, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T17:15:18,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/3897501f3af549d8aedc91742e4e38f6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/3897501f3af549d8aedc91742e4e38f6 2024-12-02T17:15:18,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/3897501f3af549d8aedc91742e4e38f6, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T17:15:18,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 189a4a1fc2d783a754e60fdd039aa718 in 1357ms, sequenceid=340, compaction requested=true 2024-12-02T17:15:18,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:18,630 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:18,631 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:18,631 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:18,631 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,632 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ef8f1b7f486e4142b3f65af4a5250ea5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=139.6 K 2024-12-02T17:15:18,632 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,632 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ef8f1b7f486e4142b3f65af4a5250ea5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2] 2024-12-02T17:15:18,632 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef8f1b7f486e4142b3f65af4a5250ea5, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733159715146 2024-12-02T17:15:18,633 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0347e9c513424454bc8032963c086978, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733159715307 2024-12-02T17:15:18,633 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33045c7876d240989ab95b87a98892e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733159715724 2024-12-02T17:15:18,633 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 904cfcf4c2474ba9891d36411f3e0cf2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159716921 2024-12-02T17:15:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:18,641 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:18,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:18,643 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:18,643 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:18,643 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,643 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/21f0bf081e0147a59c1cec1888636650, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/43f513ccc8ce446e8e61ea38bdc1f77c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/2d0ad9a925c14844b7188b081d906861, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/595331ce8667426c9e36716f2e5e21a5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.6 K 2024-12-02T17:15:18,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:18,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:18,643 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 21f0bf081e0147a59c1cec1888636650, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733159715146 2024-12-02T17:15:18,644 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f513ccc8ce446e8e61ea38bdc1f77c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733159715657 2024-12-02T17:15:18,644 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d0ad9a925c14844b7188b081d906861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733159715724 2024-12-02T17:15:18,646 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 595331ce8667426c9e36716f2e5e21a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159716921 2024-12-02T17:15:18,648 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:18,663 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412020f165a6a6bc549178983336885d54143_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:18,666 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412020f165a6a6bc549178983336885d54143_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:18,666 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020f165a6a6bc549178983336885d54143_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:18,672 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:18,673 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/58f49cda2e124122960eff88bca485f0 is 50, key is test_row_0/B:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:18,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742051_1227 (size=4469) 2024-12-02T17:15:18,688 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#192 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:18,688 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/674ee8db32a8469e81da4e48c1afed14 is 175, key is test_row_0/A:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742052_1228 (size=12949) 2024-12-02T17:15:18,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:15:18,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742053_1229 (size=31903) 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:18,722 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/674ee8db32a8469e81da4e48c1afed14 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/674ee8db32a8469e81da4e48c1afed14 2024-12-02T17:15:18,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022109e389129d4b76835627b928ab6ec5_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159717327/Put/seqid=0 2024-12-02T17:15:18,729 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into 674ee8db32a8469e81da4e48c1afed14(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:18,729 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:18,729 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=12, startTime=1733159718630; duration=0sec 2024-12-02T17:15:18,729 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:18,729 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:18,729 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:18,730 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:18,730 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:18,730 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:18,730 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e4f5b4ea622347dc852b28dc35d77c49, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/91550c9b6232446a93abe618a83ea0a1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/d07e476967cb438fbe23c6a9f1b9ad0c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/3897501f3af549d8aedc91742e4e38f6] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.6 K 2024-12-02T17:15:18,731 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f5b4ea622347dc852b28dc35d77c49, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733159715146 2024-12-02T17:15:18,731 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91550c9b6232446a93abe618a83ea0a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733159715657 2024-12-02T17:15:18,731 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d07e476967cb438fbe23c6a9f1b9ad0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733159715724 2024-12-02T17:15:18,732 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3897501f3af549d8aedc91742e4e38f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159716921 2024-12-02T17:15:18,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742054_1230 (size=12454) 2024-12-02T17:15:18,751 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:18,752 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ae25276d89c8412f94cdf0f38e7fa129 is 50, key is test_row_0/C:col10/1733159717269/Put/seqid=0 2024-12-02T17:15:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742055_1231 (size=12949) 2024-12-02T17:15:18,774 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ae25276d89c8412f94cdf0f38e7fa129 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ae25276d89c8412f94cdf0f38e7fa129 2024-12-02T17:15:18,782 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into ae25276d89c8412f94cdf0f38e7fa129(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:18,782 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:18,782 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=12, startTime=1733159718642; duration=0sec 2024-12-02T17:15:18,782 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:18,782 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:19,101 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/58f49cda2e124122960eff88bca485f0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/58f49cda2e124122960eff88bca485f0 2024-12-02T17:15:19,111 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into 58f49cda2e124122960eff88bca485f0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:19,111 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:19,111 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=12, startTime=1733159718641; duration=0sec 2024-12-02T17:15:19,111 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:19,111 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:19,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:19,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022109e389129d4b76835627b928ab6ec5_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022109e389129d4b76835627b928ab6ec5_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/41bf60cbbd7c4956b86ab0f754137de5, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/41bf60cbbd7c4956b86ab0f754137de5 is 175, key is test_row_0/A:col10/1733159717327/Put/seqid=0 2024-12-02T17:15:19,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742056_1232 (size=31255) 2024-12-02T17:15:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:19,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:19,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159779507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159779512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159779512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,554 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/41bf60cbbd7c4956b86ab0f754137de5 2024-12-02T17:15:19,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/b8927711fb034b08a43d655b40a7d0b3 is 50, key is test_row_0/B:col10/1733159717327/Put/seqid=0 2024-12-02T17:15:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159779617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159779628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159779629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742057_1233 (size=12301) 2024-12-02T17:15:19,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,634 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/b8927711fb034b08a43d655b40a7d0b3 2024-12-02T17:15:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/2cb98d74666345ff8a7fdfed29170d2b is 50, key is test_row_0/C:col10/1733159717327/Put/seqid=0 2024-12-02T17:15:19,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742058_1234 (size=12301) 2024-12-02T17:15:19,698 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/2cb98d74666345ff8a7fdfed29170d2b 2024-12-02T17:15:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/41bf60cbbd7c4956b86ab0f754137de5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5 2024-12-02T17:15:19,712 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5, entries=150, sequenceid=359, filesize=30.5 K 2024-12-02T17:15:19,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/b8927711fb034b08a43d655b40a7d0b3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b8927711fb034b08a43d655b40a7d0b3 2024-12-02T17:15:19,733 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b8927711fb034b08a43d655b40a7d0b3, entries=150, sequenceid=359, filesize=12.0 K 2024-12-02T17:15:19,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/2cb98d74666345ff8a7fdfed29170d2b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/2cb98d74666345ff8a7fdfed29170d2b 2024-12-02T17:15:19,744 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/2cb98d74666345ff8a7fdfed29170d2b, entries=150, sequenceid=359, filesize=12.0 K 2024-12-02T17:15:19,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 189a4a1fc2d783a754e60fdd039aa718 in 1038ms, sequenceid=359, compaction requested=false 2024-12-02T17:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-02T17:15:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-02T17:15:19,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-02T17:15:19,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4260 sec 2024-12-02T17:15:19,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4320 sec 2024-12-02T17:15:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:19,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:15:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:19,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:19,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:19,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026e11bb3bf51543489eb632378896dfd1_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:19,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159779898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159779901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159779904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:19,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742059_1235 (size=14994) 2024-12-02T17:15:19,915 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:19,922 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026e11bb3bf51543489eb632378896dfd1_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026e11bb3bf51543489eb632378896dfd1_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:19,923 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/6afbc420fa314bc598dd4d2c3fa5036a, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:19,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/6afbc420fa314bc598dd4d2c3fa5036a is 175, key is test_row_0/A:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:19,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742060_1236 (size=39949) 2024-12-02T17:15:20,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159780007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159780017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159780017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159780217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159780227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159780227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,343 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/6afbc420fa314bc598dd4d2c3fa5036a 2024-12-02T17:15:20,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/bfeaa02309394e4aa27834914d9c5180 is 50, key is test_row_0/B:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742061_1237 (size=12301) 2024-12-02T17:15:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159780527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159780537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159780537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:20,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/bfeaa02309394e4aa27834914d9c5180 2024-12-02T17:15:20,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/4609512124ee40f7b2098a07c444e9ad is 50, key is test_row_0/C:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:20,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742062_1238 (size=12301) 2024-12-02T17:15:20,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/4609512124ee40f7b2098a07c444e9ad 2024-12-02T17:15:20,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/6afbc420fa314bc598dd4d2c3fa5036a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a 2024-12-02T17:15:20,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a, entries=200, sequenceid=377, filesize=39.0 K 2024-12-02T17:15:20,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/bfeaa02309394e4aa27834914d9c5180 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bfeaa02309394e4aa27834914d9c5180 2024-12-02T17:15:20,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bfeaa02309394e4aa27834914d9c5180, entries=150, sequenceid=377, filesize=12.0 K 2024-12-02T17:15:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/4609512124ee40f7b2098a07c444e9ad as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/4609512124ee40f7b2098a07c444e9ad 2024-12-02T17:15:20,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/4609512124ee40f7b2098a07c444e9ad, entries=150, sequenceid=377, filesize=12.0 K 2024-12-02T17:15:20,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 189a4a1fc2d783a754e60fdd039aa718 in 1048ms, sequenceid=377, compaction requested=true 2024-12-02T17:15:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:20,890 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:20,891 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:20,892 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:20,892 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:20,892 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:20,892 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/674ee8db32a8469e81da4e48c1afed14, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=100.7 K 2024-12-02T17:15:20,892 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:20,892 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/674ee8db32a8469e81da4e48c1afed14, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a] 2024-12-02T17:15:20,893 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674ee8db32a8469e81da4e48c1afed14, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159716921 2024-12-02T17:15:20,893 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:20,893 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:20,893 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41bf60cbbd7c4956b86ab0f754137de5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159717316 2024-12-02T17:15:20,893 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:20,893 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/58f49cda2e124122960eff88bca485f0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b8927711fb034b08a43d655b40a7d0b3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bfeaa02309394e4aa27834914d9c5180] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=36.7 K 2024-12-02T17:15:20,894 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f49cda2e124122960eff88bca485f0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159716921 2024-12-02T17:15:20,894 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6afbc420fa314bc598dd4d2c3fa5036a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733159719503 2024-12-02T17:15:20,894 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b8927711fb034b08a43d655b40a7d0b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159717316 2024-12-02T17:15:20,895 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting bfeaa02309394e4aa27834914d9c5180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733159719503 2024-12-02T17:15:20,909 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:20,911 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120201c02eb1a40c457ca81e23c9cbe4cc2c_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:20,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120201c02eb1a40c457ca81e23c9cbe4cc2c_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:20,914 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120201c02eb1a40c457ca81e23c9cbe4cc2c_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:20,918 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:20,919 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5d0a6c78ca494910baebf16438c2fb28 is 50, key is test_row_0/B:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:20,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742064_1240 (size=13051) 2024-12-02T17:15:20,941 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5d0a6c78ca494910baebf16438c2fb28 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5d0a6c78ca494910baebf16438c2fb28 2024-12-02T17:15:20,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742063_1239 (size=4469) 2024-12-02T17:15:20,947 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#201 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:20,950 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into 5d0a6c78ca494910baebf16438c2fb28(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:20,950 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:20,950 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=13, startTime=1733159720890; duration=0sec 2024-12-02T17:15:20,950 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:20,950 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:20,950 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:20,951 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/aea64e81539f4d069e0da57069d47bb4 is 175, key is test_row_0/A:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:20,951 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:20,952 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:20,952 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:20,952 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ae25276d89c8412f94cdf0f38e7fa129, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/2cb98d74666345ff8a7fdfed29170d2b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/4609512124ee40f7b2098a07c444e9ad] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=36.7 K 2024-12-02T17:15:20,952 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ae25276d89c8412f94cdf0f38e7fa129, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159716921 2024-12-02T17:15:20,953 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb98d74666345ff8a7fdfed29170d2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733159717316 2024-12-02T17:15:20,953 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4609512124ee40f7b2098a07c444e9ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733159719503 2024-12-02T17:15:20,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742065_1241 (size=32005) 2024-12-02T17:15:20,990 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:20,990 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/8eaa97ce6c4f4633aad243153cfcf827 is 50, key is test_row_0/C:col10/1733159719503/Put/seqid=0 2024-12-02T17:15:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742066_1242 (size=13051) 2024-12-02T17:15:21,043 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/8eaa97ce6c4f4633aad243153cfcf827 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/8eaa97ce6c4f4633aad243153cfcf827 2024-12-02T17:15:21,094 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into 8eaa97ce6c4f4633aad243153cfcf827(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:21,094 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:21,094 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=13, startTime=1733159720890; duration=0sec 2024-12-02T17:15:21,094 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:21,094 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:21,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:15:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:21,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:21,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a2a964c11bab4abc923eafd21ea1b60a_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159721054/Put/seqid=0 2024-12-02T17:15:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159781135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159781139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159781140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742067_1243 (size=14994) 2024-12-02T17:15:21,157 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:21,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a2a964c11bab4abc923eafd21ea1b60a_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a2a964c11bab4abc923eafd21ea1b60a_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:21,170 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/dd800589fa234b8c838b812c7092f172, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:21,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/dd800589fa234b8c838b812c7092f172 is 175, key is test_row_0/A:col10/1733159721054/Put/seqid=0 2024-12-02T17:15:21,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742068_1244 (size=39949) 2024-12-02T17:15:21,225 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/dd800589fa234b8c838b812c7092f172 2024-12-02T17:15:21,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/589cf7d09f874a3f9c70603fa00a9a33 is 50, key is test_row_0/B:col10/1733159721054/Put/seqid=0 2024-12-02T17:15:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159781247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159781273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159781273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742069_1245 (size=12301) 2024-12-02T17:15:21,407 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/aea64e81539f4d069e0da57069d47bb4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/aea64e81539f4d069e0da57069d47bb4 2024-12-02T17:15:21,413 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into aea64e81539f4d069e0da57069d47bb4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:21,413 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:21,413 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=13, startTime=1733159720890; duration=0sec 2024-12-02T17:15:21,414 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:21,414 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:21,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159781466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159781487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159781487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T17:15:21,498 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:21,501 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-02T17:15:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T17:15:21,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:21,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:21,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T17:15:21,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T17:15:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:21,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:21,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:21,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/589cf7d09f874a3f9c70603fa00a9a33 2024-12-02T17:15:21,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/e31cd08fec90431a848efef83b3c0340 is 50, key is test_row_0/C:col10/1733159721054/Put/seqid=0 2024-12-02T17:15:21,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742070_1246 (size=12301) 2024-12-02T17:15:21,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/e31cd08fec90431a848efef83b3c0340 2024-12-02T17:15:21,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159781777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/dd800589fa234b8c838b812c7092f172 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172 2024-12-02T17:15:21,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172, entries=200, sequenceid=399, filesize=39.0 K 2024-12-02T17:15:21,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159781797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/589cf7d09f874a3f9c70603fa00a9a33 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/589cf7d09f874a3f9c70603fa00a9a33 2024-12-02T17:15:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159781807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:21,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:21,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T17:15:21,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:21,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:21,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T17:15:21,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/589cf7d09f874a3f9c70603fa00a9a33, entries=150, sequenceid=399, filesize=12.0 K 2024-12-02T17:15:21,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/e31cd08fec90431a848efef83b3c0340 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e31cd08fec90431a848efef83b3c0340 2024-12-02T17:15:21,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e31cd08fec90431a848efef83b3c0340, entries=150, sequenceid=399, filesize=12.0 K 2024-12-02T17:15:21,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 189a4a1fc2d783a754e60fdd039aa718 in 734ms, sequenceid=399, compaction requested=false 2024-12-02T17:15:21,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:21,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-02T17:15:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:21,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:22,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028be5a424233349f3bc143c20eb117386_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159721125/Put/seqid=0 2024-12-02T17:15:22,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742071_1247 (size=12454) 2024-12-02T17:15:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T17:15:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:22,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159782360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159782367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159782369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159782485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159782485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159782493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,525 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028be5a424233349f3bc143c20eb117386_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028be5a424233349f3bc143c20eb117386_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9d4e92cde01b4c6aaaaa6aadf34977a7, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:22,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9d4e92cde01b4c6aaaaa6aadf34977a7 is 175, key is test_row_0/A:col10/1733159721125/Put/seqid=0 2024-12-02T17:15:22,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742072_1248 (size=31255) 2024-12-02T17:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T17:15:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159782700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159782706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159782706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:22,980 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9d4e92cde01b4c6aaaaa6aadf34977a7 2024-12-02T17:15:23,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/0051f7ea78404a1fb7f1c4035f48ecf2 is 50, key is test_row_0/B:col10/1733159721125/Put/seqid=0 2024-12-02T17:15:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159783007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159783017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159783017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742073_1249 (size=12301) 2024-12-02T17:15:23,041 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/0051f7ea78404a1fb7f1c4035f48ecf2 2024-12-02T17:15:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/c300ca852c9f4a8da4a607ddd4615b18 is 50, key is test_row_0/C:col10/1733159721125/Put/seqid=0 2024-12-02T17:15:23,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742074_1250 (size=12301) 2024-12-02T17:15:23,475 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/c300ca852c9f4a8da4a607ddd4615b18 2024-12-02T17:15:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9d4e92cde01b4c6aaaaa6aadf34977a7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7 2024-12-02T17:15:23,503 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7, entries=150, sequenceid=412, filesize=30.5 K 2024-12-02T17:15:23,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/0051f7ea78404a1fb7f1c4035f48ecf2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/0051f7ea78404a1fb7f1c4035f48ecf2 2024-12-02T17:15:23,510 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/0051f7ea78404a1fb7f1c4035f48ecf2, entries=150, sequenceid=412, filesize=12.0 K 2024-12-02T17:15:23,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/c300ca852c9f4a8da4a607ddd4615b18 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c300ca852c9f4a8da4a607ddd4615b18 2024-12-02T17:15:23,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c300ca852c9f4a8da4a607ddd4615b18, entries=150, sequenceid=412, filesize=12.0 K 2024-12-02T17:15:23,519 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 189a4a1fc2d783a754e60fdd039aa718 in 1550ms, sequenceid=412, compaction requested=true 2024-12-02T17:15:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-02T17:15:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-02T17:15:23,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T17:15:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:23,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:23,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-02T17:15:23,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0150 sec 2024-12-02T17:15:23,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.0260 sec 2024-12-02T17:15:23,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120276b458f1b10f43468995bd79cfbf2d62_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:23,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159783557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159783560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159783561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742075_1251 (size=14994) 2024-12-02T17:15:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T17:15:23,657 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:23,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-02T17:15:23,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:23,663 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:23,665 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:23,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:23,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159783677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159783677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159783677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:23,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:23,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:23,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:23,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:23,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159783897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159783909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159783909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:23,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:23,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:23,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:23,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:23,985 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120276b458f1b10f43468995bd79cfbf2d62_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120276b458f1b10f43468995bd79cfbf2d62_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:23,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/c34153c3ac434679a1c546754760c0b1, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:23,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/c34153c3ac434679a1c546754760c0b1 is 175, key is test_row_0/A:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:23,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742076_1252 (size=39949) 2024-12-02T17:15:23,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/c34153c3ac434679a1c546754760c0b1 2024-12-02T17:15:24,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5600bacde67247ba8214fb94d0d4bb0c is 50, key is test_row_0/B:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:24,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742077_1253 (size=12301) 2024-12-02T17:15:24,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5600bacde67247ba8214fb94d0d4bb0c 2024-12-02T17:15:24,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/c44bc72aef184e62ac255057098f86a0 is 50, key is test_row_0/C:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742078_1254 (size=12301) 2024-12-02T17:15:24,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/c44bc72aef184e62ac255057098f86a0 2024-12-02T17:15:24,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/c34153c3ac434679a1c546754760c0b1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1 2024-12-02T17:15:24,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:24,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:24,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1, entries=200, sequenceid=433, filesize=39.0 K 2024-12-02T17:15:24,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/5600bacde67247ba8214fb94d0d4bb0c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5600bacde67247ba8214fb94d0d4bb0c 2024-12-02T17:15:24,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5600bacde67247ba8214fb94d0d4bb0c, entries=150, sequenceid=433, filesize=12.0 K 2024-12-02T17:15:24,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/c44bc72aef184e62ac255057098f86a0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c44bc72aef184e62ac255057098f86a0 2024-12-02T17:15:24,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c44bc72aef184e62ac255057098f86a0, entries=150, sequenceid=433, filesize=12.0 K 2024-12-02T17:15:24,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 189a4a1fc2d783a754e60fdd039aa718 in 687ms, sequenceid=433, compaction requested=true 2024-12-02T17:15:24,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:24,211 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:24,211 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:24,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:24,212 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:24,212 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:24,212 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,212 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/aea64e81539f4d069e0da57069d47bb4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=139.8 K 2024-12-02T17:15:24,212 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,212 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/aea64e81539f4d069e0da57069d47bb4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1] 2024-12-02T17:15:24,213 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:24,213 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:24,213 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,213 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5d0a6c78ca494910baebf16438c2fb28, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/589cf7d09f874a3f9c70603fa00a9a33, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/0051f7ea78404a1fb7f1c4035f48ecf2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5600bacde67247ba8214fb94d0d4bb0c] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.8 K 2024-12-02T17:15:24,217 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea64e81539f4d069e0da57069d47bb4, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733159719503 2024-12-02T17:15:24,217 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0a6c78ca494910baebf16438c2fb28, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733159719503 2024-12-02T17:15:24,218 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd800589fa234b8c838b812c7092f172, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733159719900 2024-12-02T17:15:24,218 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 589cf7d09f874a3f9c70603fa00a9a33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733159719900 2024-12-02T17:15:24,218 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d4e92cde01b4c6aaaaa6aadf34977a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733159721125 2024-12-02T17:15:24,219 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c34153c3ac434679a1c546754760c0b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733159722345 2024-12-02T17:15:24,219 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0051f7ea78404a1fb7f1c4035f48ecf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733159721125 2024-12-02T17:15:24,219 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5600bacde67247ba8214fb94d0d4bb0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733159722363 2024-12-02T17:15:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:24,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:24,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:24,237 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:24,241 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:24,242 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/29ffc91597e64b86a1d7f5265f9b91c1 is 50, key is test_row_0/B:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:24,245 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b9faa818fd3041f3804d43c10c6a1e77_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:24,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T17:15:24,248 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b9faa818fd3041f3804d43c10c6a1e77_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:24,248 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b9faa818fd3041f3804d43c10c6a1e77_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:24,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026240ce0024e24f25aa5a606da5b92c4d_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159724218/Put/seqid=0 2024-12-02T17:15:24,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742079_1255 (size=13187) 2024-12-02T17:15:24,281 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/29ffc91597e64b86a1d7f5265f9b91c1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/29ffc91597e64b86a1d7f5265f9b91c1 2024-12-02T17:15:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159784275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159784277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159784278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,287 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into 29ffc91597e64b86a1d7f5265f9b91c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:24,287 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:24,287 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=12, startTime=1733159724211; duration=0sec 2024-12-02T17:15:24,287 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:24,287 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:24,287 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:24,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,296 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:24,296 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:24,296 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,296 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/8eaa97ce6c4f4633aad243153cfcf827, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e31cd08fec90431a848efef83b3c0340, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c300ca852c9f4a8da4a607ddd4615b18, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c44bc72aef184e62ac255057098f86a0] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.8 K 2024-12-02T17:15:24,296 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eaa97ce6c4f4633aad243153cfcf827, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733159719503 2024-12-02T17:15:24,311 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e31cd08fec90431a848efef83b3c0340, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733159719900 2024-12-02T17:15:24,311 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c300ca852c9f4a8da4a607ddd4615b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733159721125 2024-12-02T17:15:24,312 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c44bc72aef184e62ac255057098f86a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733159722363 2024-12-02T17:15:24,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742080_1256 (size=4469) 2024-12-02T17:15:24,329 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#213 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:24,330 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/6ff521729d6e47d09a271d89747c784d is 175, key is test_row_0/A:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742081_1257 (size=14994) 2024-12-02T17:15:24,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:24,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026240ce0024e24f25aa5a606da5b92c4d_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026240ce0024e24f25aa5a606da5b92c4d_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:24,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/e22bc771cd3e47c3a09112bf0dc8daca, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:24,340 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:24,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/e22bc771cd3e47c3a09112bf0dc8daca is 175, key is test_row_0/A:col10/1733159724218/Put/seqid=0 2024-12-02T17:15:24,341 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6813d3fe1be149ed9e96ee9a55e413ae is 50, key is test_row_0/C:col10/1733159722367/Put/seqid=0 2024-12-02T17:15:24,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742082_1258 (size=32141) 2024-12-02T17:15:24,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742083_1259 (size=13187) 2024-12-02T17:15:24,381 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6813d3fe1be149ed9e96ee9a55e413ae as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6813d3fe1be149ed9e96ee9a55e413ae 2024-12-02T17:15:24,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742084_1260 (size=39949) 2024-12-02T17:15:24,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/e22bc771cd3e47c3a09112bf0dc8daca 2024-12-02T17:15:24,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159784393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159784393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159784394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,401 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into 6813d3fe1be149ed9e96ee9a55e413ae(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:24,401 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:24,401 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=12, startTime=1733159724211; duration=0sec 2024-12-02T17:15:24,401 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:24,401 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:24,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/d0a34198893c4197932a4022c0b1aff2 is 50, key is test_row_0/B:col10/1733159724218/Put/seqid=0 2024-12-02T17:15:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742085_1261 (size=12301) 2024-12-02T17:15:24,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/d0a34198893c4197932a4022c0b1aff2 2024-12-02T17:15:24,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ede86bc322de4f9595c953300b9ce73a is 50, key is test_row_0/C:col10/1733159724218/Put/seqid=0 2024-12-02T17:15:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742086_1262 (size=12301) 2024-12-02T17:15:24,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159784608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159784608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159784608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:24,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:24,770 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/6ff521729d6e47d09a271d89747c784d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6ff521729d6e47d09a271d89747c784d 2024-12-02T17:15:24,783 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into 6ff521729d6e47d09a271d89747c784d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:24,783 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:24,783 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=12, startTime=1733159724210; duration=0sec 2024-12-02T17:15:24,783 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:24,783 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:24,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ede86bc322de4f9595c953300b9ce73a 2024-12-02T17:15:24,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/e22bc771cd3e47c3a09112bf0dc8daca as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca 2024-12-02T17:15:24,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca, entries=200, sequenceid=445, filesize=39.0 K 2024-12-02T17:15:24,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/d0a34198893c4197932a4022c0b1aff2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d0a34198893c4197932a4022c0b1aff2 2024-12-02T17:15:24,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d0a34198893c4197932a4022c0b1aff2, entries=150, sequenceid=445, filesize=12.0 K 2024-12-02T17:15:24,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/ede86bc322de4f9595c953300b9ce73a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ede86bc322de4f9595c953300b9ce73a 2024-12-02T17:15:24,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ede86bc322de4f9595c953300b9ce73a, entries=150, sequenceid=445, filesize=12.0 K 2024-12-02T17:15:24,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 189a4a1fc2d783a754e60fdd039aa718 in 684ms, sequenceid=445, compaction requested=false 2024-12-02T17:15:24,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:24,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:24,913 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:24,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202309210d0cdd142d29fd8bef82686551f_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159724275/Put/seqid=0 2024-12-02T17:15:24,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. as already flushing 2024-12-02T17:15:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:24,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742087_1263 (size=12454) 2024-12-02T17:15:24,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159784947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159784950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159784951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159785057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159785057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159785067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159785266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159785266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159785277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:25,345 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202309210d0cdd142d29fd8bef82686551f_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202309210d0cdd142d29fd8bef82686551f_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/196c5d92f0f44cafa96ff34b88bf9201, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/196c5d92f0f44cafa96ff34b88bf9201 is 175, key is test_row_0/A:col10/1733159724275/Put/seqid=0 2024-12-02T17:15:25,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742088_1264 (size=31255) 2024-12-02T17:15:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159785578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159785581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159785588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:25,717 INFO [Thread-742 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-742. 2024-12-02T17:15:25,717 DEBUG [Thread-742 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:25,718 DEBUG [Thread-742 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,718 DEBUG [Thread-742 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,718 INFO [Thread-748 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-748. 2024-12-02T17:15:25,718 DEBUG [Thread-748 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:25,719 DEBUG [Thread-748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,719 DEBUG [Thread-748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:25,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:25,721 INFO [Thread-744 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-744. 2024-12-02T17:15:25,721 DEBUG [Thread-744 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:25,721 DEBUG [Thread-744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,721 DEBUG [Thread-744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:25,727 INFO [Thread-746 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-746. 2024-12-02T17:15:25,727 DEBUG [Thread-746 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:25,727 DEBUG [Thread-746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:25,727 DEBUG [Thread-746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:25,774 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=468, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/196c5d92f0f44cafa96ff34b88bf9201 2024-12-02T17:15:25,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/502c58aeb3404df3a3da81772c7bb7e1 is 50, key is test_row_0/B:col10/1733159724275/Put/seqid=0 2024-12-02T17:15:25,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742089_1265 (size=12301) 2024-12-02T17:15:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59588 deadline: 1733159786087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59618 deadline: 1733159786096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:26,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59576 deadline: 1733159786107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:15:26,190 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/502c58aeb3404df3a3da81772c7bb7e1 2024-12-02T17:15:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6013b61c4450428aa09c7aba7ade2fed is 50, key is test_row_0/C:col10/1733159724275/Put/seqid=0 2024-12-02T17:15:26,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742090_1266 (size=12301) 2024-12-02T17:15:26,622 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6013b61c4450428aa09c7aba7ade2fed 2024-12-02T17:15:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/196c5d92f0f44cafa96ff34b88bf9201 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201 2024-12-02T17:15:26,630 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201, entries=150, sequenceid=468, filesize=30.5 K 2024-12-02T17:15:26,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/502c58aeb3404df3a3da81772c7bb7e1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/502c58aeb3404df3a3da81772c7bb7e1 2024-12-02T17:15:26,635 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/502c58aeb3404df3a3da81772c7bb7e1, entries=150, sequenceid=468, filesize=12.0 K 2024-12-02T17:15:26,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/6013b61c4450428aa09c7aba7ade2fed as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6013b61c4450428aa09c7aba7ade2fed 2024-12-02T17:15:26,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6013b61c4450428aa09c7aba7ade2fed, entries=150, sequenceid=468, filesize=12.0 K 2024-12-02T17:15:26,640 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 189a4a1fc2d783a754e60fdd039aa718 in 1727ms, sequenceid=468, compaction requested=true 2024-12-02T17:15:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:26,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-02T17:15:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-02T17:15:26,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-02T17:15:26,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9760 sec 2024-12-02T17:15:26,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.9860 sec 2024-12-02T17:15:27,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:27,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:15:27,098 INFO [Thread-739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-739. 2024-12-02T17:15:27,098 DEBUG [Thread-739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:27,098 DEBUG [Thread-739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:27,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:27,099 DEBUG [Thread-739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:27,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:27,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:27,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e271e2cfeb6e4cc59991a81e68a27eb2_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:27,108 INFO [Thread-731 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-731. 2024-12-02T17:15:27,108 DEBUG [Thread-731 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:27,108 DEBUG [Thread-731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:27,108 DEBUG [Thread-731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:27,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:27,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742091_1267 (size=12454) 2024-12-02T17:15:27,118 INFO [Thread-735 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-735. 2024-12-02T17:15:27,118 DEBUG [Thread-735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:27,118 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:27,118 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:27,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:27,511 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:27,516 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202e271e2cfeb6e4cc59991a81e68a27eb2_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e271e2cfeb6e4cc59991a81e68a27eb2_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:27,517 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/4ab7411f03ec4a05af5084b4f6c620c8, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:27,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/4ab7411f03ec4a05af5084b4f6c620c8 is 175, key is test_row_0/A:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:27,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742092_1268 (size=31255) 2024-12-02T17:15:27,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=481, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/4ab7411f03ec4a05af5084b4f6c620c8 2024-12-02T17:15:27,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/c88adcded19945bf91695e303c7a37b5 is 50, key is test_row_0/B:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:27,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742093_1269 (size=12301) 2024-12-02T17:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T17:15:27,837 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:27,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/c88adcded19945bf91695e303c7a37b5 2024-12-02T17:15:27,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fb9af4a2f4344b44bc550c255c9449ab is 50, key is test_row_0/C:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742094_1270 (size=12301) 2024-12-02T17:15:27,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fb9af4a2f4344b44bc550c255c9449ab 2024-12-02T17:15:27,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/4ab7411f03ec4a05af5084b4f6c620c8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8 2024-12-02T17:15:27,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8, entries=150, sequenceid=481, filesize=30.5 K 2024-12-02T17:15:27,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/c88adcded19945bf91695e303c7a37b5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/c88adcded19945bf91695e303c7a37b5 2024-12-02T17:15:27,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/c88adcded19945bf91695e303c7a37b5, entries=150, sequenceid=481, filesize=12.0 K 2024-12-02T17:15:27,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/fb9af4a2f4344b44bc550c255c9449ab as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb9af4a2f4344b44bc550c255c9449ab 2024-12-02T17:15:27,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb9af4a2f4344b44bc550c255c9449ab, entries=150, sequenceid=481, filesize=12.0 K 2024-12-02T17:15:27,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=13.42 KB/13740 for 189a4a1fc2d783a754e60fdd039aa718 in 865ms, sequenceid=481, compaction requested=true 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:27,963 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 189a4a1fc2d783a754e60fdd039aa718:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:27,963 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:27,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:27,964 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:27,964 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:27,964 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/A is initiating minor compaction (all files) 2024-12-02T17:15:27,964 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/B is initiating minor compaction (all files) 2024-12-02T17:15:27,965 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/B in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:27,965 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/A in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:27,965 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6ff521729d6e47d09a271d89747c784d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=131.4 K 2024-12-02T17:15:27,965 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/29ffc91597e64b86a1d7f5265f9b91c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d0a34198893c4197932a4022c0b1aff2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/502c58aeb3404df3a3da81772c7bb7e1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/c88adcded19945bf91695e303c7a37b5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.9 K 2024-12-02T17:15:27,965 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:27,965 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6ff521729d6e47d09a271d89747c784d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8] 2024-12-02T17:15:27,965 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ffc91597e64b86a1d7f5265f9b91c1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733159722363 2024-12-02T17:15:27,965 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff521729d6e47d09a271d89747c784d, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733159722363 2024-12-02T17:15:27,966 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a34198893c4197932a4022c0b1aff2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733159723545 2024-12-02T17:15:27,966 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22bc771cd3e47c3a09112bf0dc8daca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733159723545 2024-12-02T17:15:27,966 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 502c58aeb3404df3a3da81772c7bb7e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733159724254 2024-12-02T17:15:27,966 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 196c5d92f0f44cafa96ff34b88bf9201, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733159724254 2024-12-02T17:15:27,966 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c88adcded19945bf91695e303c7a37b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733159724936 2024-12-02T17:15:27,967 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ab7411f03ec4a05af5084b4f6c620c8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733159724936 2024-12-02T17:15:27,985 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#B#compaction#225 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:27,985 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/b653b1eaf74847c6b6c3beed1d68c733 is 50, key is test_row_0/B:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:27,986 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:27,988 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202dfa93a4aa942401f88291679c9226229_189a4a1fc2d783a754e60fdd039aa718 store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:27,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742095_1271 (size=13323) 2024-12-02T17:15:28,004 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/b653b1eaf74847c6b6c3beed1d68c733 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b653b1eaf74847c6b6c3beed1d68c733 2024-12-02T17:15:28,011 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/B of 189a4a1fc2d783a754e60fdd039aa718 into b653b1eaf74847c6b6c3beed1d68c733(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:28,011 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:28,011 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/B, priority=12, startTime=1733159727963; duration=0sec 2024-12-02T17:15:28,011 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:28,011 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:B 2024-12-02T17:15:28,011 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:28,013 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:28,013 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 189a4a1fc2d783a754e60fdd039aa718/C is initiating minor compaction (all files) 2024-12-02T17:15:28,013 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 189a4a1fc2d783a754e60fdd039aa718/C in TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:28,013 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6813d3fe1be149ed9e96ee9a55e413ae, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ede86bc322de4f9595c953300b9ce73a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6013b61c4450428aa09c7aba7ade2fed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb9af4a2f4344b44bc550c255c9449ab] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp, totalSize=48.9 K 2024-12-02T17:15:28,013 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6813d3fe1be149ed9e96ee9a55e413ae, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733159722363 2024-12-02T17:15:28,014 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ede86bc322de4f9595c953300b9ce73a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1733159723545 2024-12-02T17:15:28,015 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6013b61c4450428aa09c7aba7ade2fed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733159724254 2024-12-02T17:15:28,015 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9af4a2f4344b44bc550c255c9449ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733159724936 2024-12-02T17:15:28,034 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202dfa93a4aa942401f88291679c9226229_189a4a1fc2d783a754e60fdd039aa718, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:28,034 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#C#compaction#227 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:28,034 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dfa93a4aa942401f88291679c9226229_189a4a1fc2d783a754e60fdd039aa718 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:28,035 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/0b3e9776b1634632aeddba30fddab42a is 50, key is test_row_0/C:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:28,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742097_1273 (size=13323) 2024-12-02T17:15:28,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742096_1272 (size=4469) 2024-12-02T17:15:28,455 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 189a4a1fc2d783a754e60fdd039aa718#A#compaction#226 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:15:28,456 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9801ffbb793445e2b607cfcbb618a4f6 is 175, key is test_row_0/A:col10/1733159724936/Put/seqid=0 2024-12-02T17:15:28,459 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/0b3e9776b1634632aeddba30fddab42a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/0b3e9776b1634632aeddba30fddab42a 2024-12-02T17:15:28,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742098_1274 (size=32277) 2024-12-02T17:15:28,464 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/C of 189a4a1fc2d783a754e60fdd039aa718 into 0b3e9776b1634632aeddba30fddab42a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:28,464 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:28,464 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/C, priority=12, startTime=1733159727963; duration=0sec 2024-12-02T17:15:28,464 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:28,464 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:C 2024-12-02T17:15:28,569 INFO [Thread-733 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-733. 2024-12-02T17:15:28,569 DEBUG [Thread-733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:28,569 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:28,569 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:28,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:28,659 INFO [Thread-737 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-737. 2024-12-02T17:15:28,660 DEBUG [Thread-737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:28,660 DEBUG [Thread-737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:28,660 DEBUG [Thread-737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2083 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1995 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 918 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2752 rows 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 932 2024-12-02T17:15:28,660 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2787 rows 2024-12-02T17:15:28,660 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:15:28,660 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:15:28,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:28,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:28,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:28,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:28,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T17:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T17:15:28,665 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159728665"}]},"ts":"1733159728665"} 2024-12-02T17:15:28,667 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T17:15:28,667 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T17:15:28,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:15:28,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, UNASSIGN}] 2024-12-02T17:15:28,670 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, UNASSIGN 2024-12-02T17:15:28,671 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:28,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, UNASSIGN because future has completed 2024-12-02T17:15:28,672 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:15:28,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:15:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T17:15:28,807 DEBUG [regionserver/3b3375d1a925:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-12-02T17:15:28,824 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:28,824 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:15:28,824 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 189a4a1fc2d783a754e60fdd039aa718, disabling compactions & flushes 2024-12-02T17:15:28,825 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:28,865 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/9801ffbb793445e2b607cfcbb618a4f6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9801ffbb793445e2b607cfcbb618a4f6 2024-12-02T17:15:28,870 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 189a4a1fc2d783a754e60fdd039aa718/A of 189a4a1fc2d783a754e60fdd039aa718 into 9801ffbb793445e2b607cfcbb618a4f6(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:28,870 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 189a4a1fc2d783a754e60fdd039aa718: 2024-12-02T17:15:28,870 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., storeName=189a4a1fc2d783a754e60fdd039aa718/A, priority=12, startTime=1733159727963; duration=0sec 2024-12-02T17:15:28,870 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:28,870 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:28,870 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 189a4a1fc2d783a754e60fdd039aa718:A 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. after waiting 0 ms 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:28,870 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 189a4a1fc2d783a754e60fdd039aa718 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=A 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=B 2024-12-02T17:15:28,870 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:28,871 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 189a4a1fc2d783a754e60fdd039aa718, store=C 2024-12-02T17:15:28,871 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:28,876 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c86de39b5534405a8f8a7219737ac44f_189a4a1fc2d783a754e60fdd039aa718 is 50, key is test_row_0/A:col10/1733159727107/Put/seqid=0 2024-12-02T17:15:28,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742099_1275 (size=12454) 2024-12-02T17:15:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T17:15:29,280 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:29,284 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c86de39b5534405a8f8a7219737ac44f_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c86de39b5534405a8f8a7219737ac44f_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:29,285 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/39be8709f4384a70af36ab5e37c61d07, store: [table=TestAcidGuarantees family=A region=189a4a1fc2d783a754e60fdd039aa718] 2024-12-02T17:15:29,286 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/39be8709f4384a70af36ab5e37c61d07 is 175, key is test_row_0/A:col10/1733159727107/Put/seqid=0 2024-12-02T17:15:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T17:15:29,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742100_1276 (size=31255) 2024-12-02T17:15:29,692 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/39be8709f4384a70af36ab5e37c61d07 2024-12-02T17:15:29,699 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/e86ac6b8a84540799e471f545b3ff68b is 50, key is test_row_0/B:col10/1733159727107/Put/seqid=0 2024-12-02T17:15:29,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742101_1277 (size=12301) 2024-12-02T17:15:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T17:15:30,103 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/e86ac6b8a84540799e471f545b3ff68b 2024-12-02T17:15:30,111 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/f4d58a897b874e2b9048fcec0260c54d is 50, key is test_row_0/C:col10/1733159727107/Put/seqid=0 2024-12-02T17:15:30,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742102_1278 (size=12301) 2024-12-02T17:15:30,515 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/f4d58a897b874e2b9048fcec0260c54d 2024-12-02T17:15:30,521 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/A/39be8709f4384a70af36ab5e37c61d07 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/39be8709f4384a70af36ab5e37c61d07 2024-12-02T17:15:30,526 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/39be8709f4384a70af36ab5e37c61d07, entries=150, sequenceid=491, filesize=30.5 K 2024-12-02T17:15:30,527 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/B/e86ac6b8a84540799e471f545b3ff68b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/e86ac6b8a84540799e471f545b3ff68b 2024-12-02T17:15:30,534 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/e86ac6b8a84540799e471f545b3ff68b, entries=150, sequenceid=491, filesize=12.0 K 2024-12-02T17:15:30,535 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/.tmp/C/f4d58a897b874e2b9048fcec0260c54d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f4d58a897b874e2b9048fcec0260c54d 2024-12-02T17:15:30,543 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f4d58a897b874e2b9048fcec0260c54d, entries=150, sequenceid=491, filesize=12.0 K 2024-12-02T17:15:30,544 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 189a4a1fc2d783a754e60fdd039aa718 in 1674ms, sequenceid=491, compaction requested=false 2024-12-02T17:15:30,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4c9379bfc4784d6d89d19a94e6dabd78, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/a4f7a294c91a47829c8ef4ae95e1d459, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5752298f15504b93b23db3d53833ee2b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ef8f1b7f486e4142b3f65af4a5250ea5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/674ee8db32a8469e81da4e48c1afed14, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/aea64e81539f4d069e0da57069d47bb4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6ff521729d6e47d09a271d89747c784d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8] to archive 2024-12-02T17:15:30,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:15:30,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9452375f9ee744ef9dc3d11f204dc149 2024-12-02T17:15:30,558 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/1fb715d07dab4a64a0200a609ebbf873 2024-12-02T17:15:30,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7613b3a269f14117a524b4af4028b603 2024-12-02T17:15:30,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7e6e5064c1cb477abff8e37e7576374b 2024-12-02T17:15:30,564 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4c9379bfc4784d6d89d19a94e6dabd78 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4c9379bfc4784d6d89d19a94e6dabd78 2024-12-02T17:15:30,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/42bdbced56fc43c3a80374ea53144e69 2024-12-02T17:15:30,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/a4f7a294c91a47829c8ef4ae95e1d459 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/a4f7a294c91a47829c8ef4ae95e1d459 2024-12-02T17:15:30,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/7cc9c7d20ebe4a3fab57e4619eb1b934 2024-12-02T17:15:30,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/18c061b104574dc8884253a9726f8ee9 2024-12-02T17:15:30,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ceeb7d9456484024b2640909d6482815 2024-12-02T17:15:30,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5e0661f86e1749a68f965bbc8bdbd6bd 2024-12-02T17:15:30,583 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5752298f15504b93b23db3d53833ee2b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/5752298f15504b93b23db3d53833ee2b 2024-12-02T17:15:30,587 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/02e9b2c53ef743aa8288fea49b05878b 2024-12-02T17:15:30,594 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/8270f99af3d3458690b1709262c762b2 2024-12-02T17:15:30,596 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ef8f1b7f486e4142b3f65af4a5250ea5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/ef8f1b7f486e4142b3f65af4a5250ea5 2024-12-02T17:15:30,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/d00a69d9c99c46928275a609b49f31fe 2024-12-02T17:15:30,599 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/0347e9c513424454bc8032963c086978 2024-12-02T17:15:30,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/33045c7876d240989ab95b87a98892e4 2024-12-02T17:15:30,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/674ee8db32a8469e81da4e48c1afed14 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/674ee8db32a8469e81da4e48c1afed14 2024-12-02T17:15:30,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/904cfcf4c2474ba9891d36411f3e0cf2 2024-12-02T17:15:30,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/41bf60cbbd7c4956b86ab0f754137de5 2024-12-02T17:15:30,606 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6afbc420fa314bc598dd4d2c3fa5036a 2024-12-02T17:15:30,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/aea64e81539f4d069e0da57069d47bb4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/aea64e81539f4d069e0da57069d47bb4 2024-12-02T17:15:30,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/dd800589fa234b8c838b812c7092f172 2024-12-02T17:15:30,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9d4e92cde01b4c6aaaaa6aadf34977a7 2024-12-02T17:15:30,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/c34153c3ac434679a1c546754760c0b1 2024-12-02T17:15:30,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6ff521729d6e47d09a271d89747c784d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/6ff521729d6e47d09a271d89747c784d 2024-12-02T17:15:30,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/e22bc771cd3e47c3a09112bf0dc8daca 2024-12-02T17:15:30,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/196c5d92f0f44cafa96ff34b88bf9201 2024-12-02T17:15:30,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/4ab7411f03ec4a05af5084b4f6c620c8 2024-12-02T17:15:30,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/86df964b6d244716b211c00324fa9e21, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/1181c8f9c22747c0a0c313040fd08d06, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/a0f2d2eb540d471f85a1c83a73085925, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/af7430839480451b979a891caa6c66e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d1a269fc88d64098a1d441c34777f781, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/9dabea3d66254daaa0dea7ef3960624b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/31237a7783fa4231a23df83962c8ec40, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/64243daf93324de1a9694a0b0eeddcd3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5cd4d6b7c87a492c99946b34f79e0597, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bf3a09f24e484978b28bd6fe8b68e0a1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/abc8adb61b884fa79efae2934a7eb033, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/520603e3a4a7481fa664be050b2bfa76, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/3c8c1af0b280418e9d1231b6c0c0dcb5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/cbba9e457e3b43e9ac22b577c268b243, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/21f0bf081e0147a59c1cec1888636650, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/17a9b6c525aa4b599261c7a4a573d8e1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/43f513ccc8ce446e8e61ea38bdc1f77c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/2d0ad9a925c14844b7188b081d906861, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/58f49cda2e124122960eff88bca485f0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/595331ce8667426c9e36716f2e5e21a5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b8927711fb034b08a43d655b40a7d0b3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5d0a6c78ca494910baebf16438c2fb28, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bfeaa02309394e4aa27834914d9c5180, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/589cf7d09f874a3f9c70603fa00a9a33, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/0051f7ea78404a1fb7f1c4035f48ecf2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/29ffc91597e64b86a1d7f5265f9b91c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5600bacde67247ba8214fb94d0d4bb0c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d0a34198893c4197932a4022c0b1aff2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/502c58aeb3404df3a3da81772c7bb7e1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/c88adcded19945bf91695e303c7a37b5] to archive 2024-12-02T17:15:30,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:15:30,623 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/86df964b6d244716b211c00324fa9e21 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/86df964b6d244716b211c00324fa9e21 2024-12-02T17:15:30,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/1181c8f9c22747c0a0c313040fd08d06 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/1181c8f9c22747c0a0c313040fd08d06 2024-12-02T17:15:30,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/a0f2d2eb540d471f85a1c83a73085925 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/a0f2d2eb540d471f85a1c83a73085925 2024-12-02T17:15:30,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/af7430839480451b979a891caa6c66e4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/af7430839480451b979a891caa6c66e4 2024-12-02T17:15:30,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d1a269fc88d64098a1d441c34777f781 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d1a269fc88d64098a1d441c34777f781 2024-12-02T17:15:30,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/9dabea3d66254daaa0dea7ef3960624b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/9dabea3d66254daaa0dea7ef3960624b 2024-12-02T17:15:30,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/31237a7783fa4231a23df83962c8ec40 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/31237a7783fa4231a23df83962c8ec40 2024-12-02T17:15:30,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/64243daf93324de1a9694a0b0eeddcd3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/64243daf93324de1a9694a0b0eeddcd3 2024-12-02T17:15:30,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5cd4d6b7c87a492c99946b34f79e0597 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5cd4d6b7c87a492c99946b34f79e0597 2024-12-02T17:15:30,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bf3a09f24e484978b28bd6fe8b68e0a1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bf3a09f24e484978b28bd6fe8b68e0a1 2024-12-02T17:15:30,639 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/abc8adb61b884fa79efae2934a7eb033 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/abc8adb61b884fa79efae2934a7eb033 2024-12-02T17:15:30,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/520603e3a4a7481fa664be050b2bfa76 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/520603e3a4a7481fa664be050b2bfa76 2024-12-02T17:15:30,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/3c8c1af0b280418e9d1231b6c0c0dcb5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/3c8c1af0b280418e9d1231b6c0c0dcb5 2024-12-02T17:15:30,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/cbba9e457e3b43e9ac22b577c268b243 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/cbba9e457e3b43e9ac22b577c268b243 2024-12-02T17:15:30,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/21f0bf081e0147a59c1cec1888636650 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/21f0bf081e0147a59c1cec1888636650 2024-12-02T17:15:30,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/17a9b6c525aa4b599261c7a4a573d8e1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/17a9b6c525aa4b599261c7a4a573d8e1 2024-12-02T17:15:30,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/43f513ccc8ce446e8e61ea38bdc1f77c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/43f513ccc8ce446e8e61ea38bdc1f77c 2024-12-02T17:15:30,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/2d0ad9a925c14844b7188b081d906861 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/2d0ad9a925c14844b7188b081d906861 2024-12-02T17:15:30,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/58f49cda2e124122960eff88bca485f0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/58f49cda2e124122960eff88bca485f0 2024-12-02T17:15:30,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/595331ce8667426c9e36716f2e5e21a5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/595331ce8667426c9e36716f2e5e21a5 2024-12-02T17:15:30,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b8927711fb034b08a43d655b40a7d0b3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b8927711fb034b08a43d655b40a7d0b3 2024-12-02T17:15:30,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5d0a6c78ca494910baebf16438c2fb28 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5d0a6c78ca494910baebf16438c2fb28 2024-12-02T17:15:30,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bfeaa02309394e4aa27834914d9c5180 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/bfeaa02309394e4aa27834914d9c5180 2024-12-02T17:15:30,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/589cf7d09f874a3f9c70603fa00a9a33 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/589cf7d09f874a3f9c70603fa00a9a33 2024-12-02T17:15:30,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/0051f7ea78404a1fb7f1c4035f48ecf2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/0051f7ea78404a1fb7f1c4035f48ecf2 2024-12-02T17:15:30,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/29ffc91597e64b86a1d7f5265f9b91c1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/29ffc91597e64b86a1d7f5265f9b91c1 2024-12-02T17:15:30,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5600bacde67247ba8214fb94d0d4bb0c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/5600bacde67247ba8214fb94d0d4bb0c 2024-12-02T17:15:30,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d0a34198893c4197932a4022c0b1aff2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/d0a34198893c4197932a4022c0b1aff2 2024-12-02T17:15:30,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/502c58aeb3404df3a3da81772c7bb7e1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/502c58aeb3404df3a3da81772c7bb7e1 2024-12-02T17:15:30,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/c88adcded19945bf91695e303c7a37b5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/c88adcded19945bf91695e303c7a37b5 2024-12-02T17:15:30,675 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb7115cc6b434dc18fe73fd67d3cbc2d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/39c13bfe32094b45aa669fd1605052d7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/5683073b21324034b3eb40110102313b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/cab394faa0d047b49ba7bdf7abe4db6b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fc4177db699c4defb6402a69110feefa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f690d1dc873a437a9bd1a1cc49f6efd0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/33ffea0ce0684eb5906400e8723c3871, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/05c54d6aed534e5a9392e689dbea022b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/b7914f2e173c4975866db2314872f16f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6e436a26fa12483fb003c921ca530407, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/609f0b63d4c0443e80a395e5e1993f76, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/a4f0dcb691114c239bff24fe19d4dc4b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ac0db473f061497f9d4a16f18400cbf7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/be1e6804f9c54416806159d83275b87e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e4f5b4ea622347dc852b28dc35d77c49, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/15bb1c01f03b4fce975f531438bacb65, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/91550c9b6232446a93abe618a83ea0a1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/d07e476967cb438fbe23c6a9f1b9ad0c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ae25276d89c8412f94cdf0f38e7fa129, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/3897501f3af549d8aedc91742e4e38f6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/2cb98d74666345ff8a7fdfed29170d2b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/8eaa97ce6c4f4633aad243153cfcf827, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/4609512124ee40f7b2098a07c444e9ad, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e31cd08fec90431a848efef83b3c0340, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c300ca852c9f4a8da4a607ddd4615b18, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6813d3fe1be149ed9e96ee9a55e413ae, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c44bc72aef184e62ac255057098f86a0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ede86bc322de4f9595c953300b9ce73a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6013b61c4450428aa09c7aba7ade2fed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb9af4a2f4344b44bc550c255c9449ab] to archive 2024-12-02T17:15:30,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:15:30,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb7115cc6b434dc18fe73fd67d3cbc2d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb7115cc6b434dc18fe73fd67d3cbc2d 2024-12-02T17:15:30,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/39c13bfe32094b45aa669fd1605052d7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/39c13bfe32094b45aa669fd1605052d7 2024-12-02T17:15:30,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/5683073b21324034b3eb40110102313b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/5683073b21324034b3eb40110102313b 2024-12-02T17:15:30,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/cab394faa0d047b49ba7bdf7abe4db6b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/cab394faa0d047b49ba7bdf7abe4db6b 2024-12-02T17:15:30,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fc4177db699c4defb6402a69110feefa to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fc4177db699c4defb6402a69110feefa 2024-12-02T17:15:30,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f690d1dc873a437a9bd1a1cc49f6efd0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f690d1dc873a437a9bd1a1cc49f6efd0 2024-12-02T17:15:30,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/33ffea0ce0684eb5906400e8723c3871 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/33ffea0ce0684eb5906400e8723c3871 2024-12-02T17:15:30,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/05c54d6aed534e5a9392e689dbea022b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/05c54d6aed534e5a9392e689dbea022b 2024-12-02T17:15:30,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/b7914f2e173c4975866db2314872f16f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/b7914f2e173c4975866db2314872f16f 2024-12-02T17:15:30,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6e436a26fa12483fb003c921ca530407 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6e436a26fa12483fb003c921ca530407 2024-12-02T17:15:30,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/609f0b63d4c0443e80a395e5e1993f76 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/609f0b63d4c0443e80a395e5e1993f76 2024-12-02T17:15:30,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/a4f0dcb691114c239bff24fe19d4dc4b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/a4f0dcb691114c239bff24fe19d4dc4b 2024-12-02T17:15:30,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ac0db473f061497f9d4a16f18400cbf7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ac0db473f061497f9d4a16f18400cbf7 2024-12-02T17:15:30,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/be1e6804f9c54416806159d83275b87e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/be1e6804f9c54416806159d83275b87e 2024-12-02T17:15:30,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e4f5b4ea622347dc852b28dc35d77c49 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e4f5b4ea622347dc852b28dc35d77c49 2024-12-02T17:15:30,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/15bb1c01f03b4fce975f531438bacb65 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/15bb1c01f03b4fce975f531438bacb65 2024-12-02T17:15:30,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/91550c9b6232446a93abe618a83ea0a1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/91550c9b6232446a93abe618a83ea0a1 2024-12-02T17:15:30,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/d07e476967cb438fbe23c6a9f1b9ad0c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/d07e476967cb438fbe23c6a9f1b9ad0c 2024-12-02T17:15:30,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ae25276d89c8412f94cdf0f38e7fa129 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ae25276d89c8412f94cdf0f38e7fa129 2024-12-02T17:15:30,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/3897501f3af549d8aedc91742e4e38f6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/3897501f3af549d8aedc91742e4e38f6 2024-12-02T17:15:30,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/2cb98d74666345ff8a7fdfed29170d2b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/2cb98d74666345ff8a7fdfed29170d2b 2024-12-02T17:15:30,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/8eaa97ce6c4f4633aad243153cfcf827 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/8eaa97ce6c4f4633aad243153cfcf827 2024-12-02T17:15:30,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/4609512124ee40f7b2098a07c444e9ad to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/4609512124ee40f7b2098a07c444e9ad 2024-12-02T17:15:30,713 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e31cd08fec90431a848efef83b3c0340 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/e31cd08fec90431a848efef83b3c0340 2024-12-02T17:15:30,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c300ca852c9f4a8da4a607ddd4615b18 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c300ca852c9f4a8da4a607ddd4615b18 2024-12-02T17:15:30,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6813d3fe1be149ed9e96ee9a55e413ae to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6813d3fe1be149ed9e96ee9a55e413ae 2024-12-02T17:15:30,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c44bc72aef184e62ac255057098f86a0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/c44bc72aef184e62ac255057098f86a0 2024-12-02T17:15:30,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ede86bc322de4f9595c953300b9ce73a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/ede86bc322de4f9595c953300b9ce73a 2024-12-02T17:15:30,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6013b61c4450428aa09c7aba7ade2fed to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/6013b61c4450428aa09c7aba7ade2fed 2024-12-02T17:15:30,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb9af4a2f4344b44bc550c255c9449ab to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/fb9af4a2f4344b44bc550c255c9449ab 2024-12-02T17:15:30,733 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/recovered.edits/494.seqid, newMaxSeqId=494, maxSeqId=4 2024-12-02T17:15:30,734 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. 2024-12-02T17:15:30,734 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 189a4a1fc2d783a754e60fdd039aa718: Waiting for close lock at 1733159728824Running coprocessor pre-close hooks at 1733159728824Disabling compacts and flushes for region at 1733159728824Disabling writes for close at 1733159728870 (+46 ms)Obtaining lock to block concurrent updates at 1733159728870Preparing flush snapshotting stores in 189a4a1fc2d783a754e60fdd039aa718 at 1733159728870Finished memstore snapshotting TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733159728871 (+1 ms)Flushing stores of TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718. at 1733159728871Flushing 189a4a1fc2d783a754e60fdd039aa718/A: creating writer at 1733159728871Flushing mob file 189a4a1fc2d783a754e60fdd039aa718/A: appending metadata at 1733159728876 (+5 ms)Flushing mob file 189a4a1fc2d783a754e60fdd039aa718/A: closing flushed file at 1733159728876Flushing 189a4a1fc2d783a754e60fdd039aa718/A: appending metadata at 1733159729285 (+409 ms)Flushing 189a4a1fc2d783a754e60fdd039aa718/A: closing flushed file at 1733159729286 (+1 ms)Flushing 189a4a1fc2d783a754e60fdd039aa718/B: creating writer at 1733159729696 (+410 ms)Flushing 189a4a1fc2d783a754e60fdd039aa718/B: appending metadata at 1733159729698 (+2 ms)Flushing 189a4a1fc2d783a754e60fdd039aa718/B: closing flushed file at 1733159729698Flushing 189a4a1fc2d783a754e60fdd039aa718/C: creating writer at 1733159730107 (+409 ms)Flushing 189a4a1fc2d783a754e60fdd039aa718/C: appending metadata at 1733159730110 (+3 ms)Flushing 189a4a1fc2d783a754e60fdd039aa718/C: closing flushed file at 1733159730110Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f612daa: reopening flushed file at 1733159730520 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54b49d91: reopening flushed file at 1733159730526 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d0733e1: reopening flushed file at 1733159730534 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 189a4a1fc2d783a754e60fdd039aa718 in 1674ms, sequenceid=491, compaction requested=false at 1733159730544 (+10 ms)Writing region close event to WAL at 1733159730729 (+185 ms)Running coprocessor post-close hooks at 1733159730734 (+5 ms)Closed at 1733159730734 2024-12-02T17:15:30,737 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,737 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=189a4a1fc2d783a754e60fdd039aa718, regionState=CLOSED 2024-12-02T17:15:30,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:15:30,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-02T17:15:30,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 189a4a1fc2d783a754e60fdd039aa718, server=3b3375d1a925,40723,1733159666211 in 2.0680 sec 2024-12-02T17:15:30,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T17:15:30,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=189a4a1fc2d783a754e60fdd039aa718, UNASSIGN in 2.0720 sec 2024-12-02T17:15:30,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-02T17:15:30,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0820 sec 2024-12-02T17:15:30,755 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159730755"}]},"ts":"1733159730755"} 2024-12-02T17:15:30,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T17:15:30,756 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T17:15:30,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0950 sec 2024-12-02T17:15:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T17:15:30,808 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:30,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T17:15:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,813 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,814 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T17:15:30,817 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,820 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/recovered.edits] 2024-12-02T17:15:30,825 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/39be8709f4384a70af36ab5e37c61d07 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/39be8709f4384a70af36ab5e37c61d07 2024-12-02T17:15:30,827 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9801ffbb793445e2b607cfcbb618a4f6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/A/9801ffbb793445e2b607cfcbb618a4f6 2024-12-02T17:15:30,830 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b653b1eaf74847c6b6c3beed1d68c733 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/b653b1eaf74847c6b6c3beed1d68c733 2024-12-02T17:15:30,832 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/e86ac6b8a84540799e471f545b3ff68b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/B/e86ac6b8a84540799e471f545b3ff68b 2024-12-02T17:15:30,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/0b3e9776b1634632aeddba30fddab42a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/0b3e9776b1634632aeddba30fddab42a 2024-12-02T17:15:30,837 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f4d58a897b874e2b9048fcec0260c54d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/C/f4d58a897b874e2b9048fcec0260c54d 2024-12-02T17:15:30,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/recovered.edits/494.seqid to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718/recovered.edits/494.seqid 2024-12-02T17:15:30,841 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,841 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T17:15:30,842 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T17:15:30,843 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T17:15:30,847 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021e2ae433461a4e159ac4682bc5c47a9a_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021e2ae433461a4e159ac4682bc5c47a9a_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,849 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022109e389129d4b76835627b928ab6ec5_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022109e389129d4b76835627b928ab6ec5_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,850 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202309210d0cdd142d29fd8bef82686551f_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202309210d0cdd142d29fd8bef82686551f_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,852 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120235d51995941f40d3b8af070d58c11d41_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120235d51995941f40d3b8af070d58c11d41_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,853 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ebe4682eaf44331aa537f43099006f6_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025ebe4682eaf44331aa537f43099006f6_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,855 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026240ce0024e24f25aa5a606da5b92c4d_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026240ce0024e24f25aa5a606da5b92c4d_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,856 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026e11bb3bf51543489eb632378896dfd1_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026e11bb3bf51543489eb632378896dfd1_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026f7ee8a58b5b4d13bc88abcf913c2ce2_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026f7ee8a58b5b4d13bc88abcf913c2ce2_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120276b458f1b10f43468995bd79cfbf2d62_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120276b458f1b10f43468995bd79cfbf2d62_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028be5a424233349f3bc143c20eb117386_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028be5a424233349f3bc143c20eb117386_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,869 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a2a964c11bab4abc923eafd21ea1b60a_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a2a964c11bab4abc923eafd21ea1b60a_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b52a9df9458f4fb49c894dc400be9a4d_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b52a9df9458f4fb49c894dc400be9a4d_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c631a299fb5448238b0097ea4e59bc23_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c631a299fb5448238b0097ea4e59bc23_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c6de9d65a2ac4719854ef00a8608ab53_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c6de9d65a2ac4719854ef00a8608ab53_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7026de2c6414630a649ed519078364e_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7026de2c6414630a649ed519078364e_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,877 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c86de39b5534405a8f8a7219737ac44f_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c86de39b5534405a8f8a7219737ac44f_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,879 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202df4f3c93ea954eae90f2c1af204e7b1d_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202df4f3c93ea954eae90f2c1af204e7b1d_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,881 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e00e8cfef3a84745af916edc50d0a87e_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e00e8cfef3a84745af916edc50d0a87e_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,882 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e271e2cfeb6e4cc59991a81e68a27eb2_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e271e2cfeb6e4cc59991a81e68a27eb2_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,884 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e454e688ed2448db9c6a246679089beb_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e454e688ed2448db9c6a246679089beb_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,888 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e5dedb6d36cd43638159d7342ab39a40_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e5dedb6d36cd43638159d7342ab39a40_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,892 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e750dff39bd749b4ad7cf71a248f5ce8_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202e750dff39bd749b4ad7cf71a248f5ce8_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,894 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f2395a1fdad64db1a62b6517aca08d34_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f2395a1fdad64db1a62b6517aca08d34_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,895 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fd8ed05c87804c9d9a6b8aced48824b3_189a4a1fc2d783a754e60fdd039aa718 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fd8ed05c87804c9d9a6b8aced48824b3_189a4a1fc2d783a754e60fdd039aa718 2024-12-02T17:15:30,896 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T17:15:30,898 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,901 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T17:15:30,904 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T17:15:30,905 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,905 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T17:15:30,906 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733159730905"}]},"ts":"9223372036854775807"} 2024-12-02T17:15:30,908 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T17:15:30,908 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 189a4a1fc2d783a754e60fdd039aa718, NAME => 'TestAcidGuarantees,,1733159702490.189a4a1fc2d783a754e60fdd039aa718.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T17:15:30,908 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T17:15:30,908 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733159730908"}]},"ts":"9223372036854775807"} 2024-12-02T17:15:30,911 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T17:15:30,912 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 103 msec 2024-12-02T17:15:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-02T17:15:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T17:15:30,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:30,944 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 246) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482944666_22 at /127.0.0.1:47670 [Waiting for operation #270] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482944666_22 at /127.0.0.1:48208 [Waiting for operation #90] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645024115_22 at /127.0.0.1:47672 [Waiting for operation #266] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645024115_22 at /127.0.0.1:38686 [Waiting for operation #291] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=908 (was 925), ProcessCount=11 (was 11), AvailableMemoryMB=3838 (was 3137) - AvailableMemoryMB LEAK? - 2024-12-02T17:15:30,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=908, ProcessCount=11, AvailableMemoryMB=3835 2024-12-02T17:15:30,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:15:30,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:15:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T17:15:30,960 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:15:30,960 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:30,965 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-12-02T17:15:30,965 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:15:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T17:15:31,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742103_1279 (size=1061) 2024-12-02T17:15:31,006 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:15:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742104_1280 (size=53) 2024-12-02T17:15:31,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:31,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cfac2600ff5eef41b88bb4f556e87599, disabling compactions & flushes 2024-12-02T17:15:31,012 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. after waiting 0 ms 2024-12-02T17:15:31,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,012 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,012 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfac2600ff5eef41b88bb4f556e87599: Waiting for close lock at 1733159731012Disabling compacts and flushes for region at 1733159731012Disabling writes for close at 1733159731012Writing region close event to WAL at 1733159731012Closed at 1733159731012 2024-12-02T17:15:31,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:15:31,014 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733159731014"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159731014"}]},"ts":"1733159731014"} 2024-12-02T17:15:31,016 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T17:15:31,017 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:15:31,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159731017"}]},"ts":"1733159731017"} 2024-12-02T17:15:31,019 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T17:15:31,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, ASSIGN}] 2024-12-02T17:15:31,020 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, ASSIGN 2024-12-02T17:15:31,021 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, ASSIGN; state=OFFLINE, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=false 2024-12-02T17:15:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T17:15:31,171 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=cfac2600ff5eef41b88bb4f556e87599, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, ASSIGN because future has completed 2024-12-02T17:15:31,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:15:31,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T17:15:31,329 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,329 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:15:31,330 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,330 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:15:31,330 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,330 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,331 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,333 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:31,333 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfac2600ff5eef41b88bb4f556e87599 columnFamilyName A 2024-12-02T17:15:31,334 DEBUG [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:31,334 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.HStore(327): Store=cfac2600ff5eef41b88bb4f556e87599/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:31,334 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,336 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:31,336 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfac2600ff5eef41b88bb4f556e87599 columnFamilyName B 2024-12-02T17:15:31,336 DEBUG [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:31,336 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.HStore(327): Store=cfac2600ff5eef41b88bb4f556e87599/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:31,337 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,338 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:15:31,338 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfac2600ff5eef41b88bb4f556e87599 columnFamilyName C 2024-12-02T17:15:31,338 DEBUG [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:15:31,338 INFO [StoreOpener-cfac2600ff5eef41b88bb4f556e87599-1 {}] regionserver.HStore(327): Store=cfac2600ff5eef41b88bb4f556e87599/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:15:31,338 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,339 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,339 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,340 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,341 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,341 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,341 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:15:31,342 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,351 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:15:31,351 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened cfac2600ff5eef41b88bb4f556e87599; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59203645, jitterRate=-0.11779694259166718}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:15:31,351 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,352 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for cfac2600ff5eef41b88bb4f556e87599: Running coprocessor pre-open hook at 1733159731330Writing region info on filesystem at 1733159731330Initializing all the Stores at 1733159731331 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159731331Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159731331Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159731331Cleaning up temporary data from old regions at 1733159731341 (+10 ms)Running coprocessor post-open hooks at 1733159731351 (+10 ms)Region opened successfully at 1733159731352 (+1 ms) 2024-12-02T17:15:31,353 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., pid=63, masterSystemTime=1733159731326 2024-12-02T17:15:31,356 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=cfac2600ff5eef41b88bb4f556e87599, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,359 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,359 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:15:31,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-02T17:15:31,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 in 186 msec 2024-12-02T17:15:31,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T17:15:31,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, ASSIGN in 343 msec 2024-12-02T17:15:31,373 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:15:31,373 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159731373"}]},"ts":"1733159731373"} 2024-12-02T17:15:31,375 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T17:15:31,376 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:15:31,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 418 msec 2024-12-02T17:15:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T17:15:31,598 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:31,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e5abc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,603 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb29393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,604 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,612 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:15:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:15:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e8fdd8a 2024-12-02T17:15:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:15:31,615 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:15:31,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,628 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d662f6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1160d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,655 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb5b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77dd910d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f851655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,664 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202a1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b989fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d40ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,671 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f27cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b9d2dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,686 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ae698f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4efb2de9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1338bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,700 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b50f7a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,702 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b97574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f84d53e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dad1786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f07c853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255e9f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,726 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:15:31,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:15:31,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:15:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:15:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:15:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc04334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:15:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:15:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:31,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56620614, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:15:31,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-02T17:15:31,742 DEBUG [Thread-1297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T17:15:31,743 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,744 DEBUG [Thread-1288 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,744 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:31,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:31,744 DEBUG [Thread-1286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,745 DEBUG [Thread-1293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,747 DEBUG [Thread-1284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,749 DEBUG [Thread-1284 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,749 DEBUG [Thread-1284 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,750 DEBUG [Thread-1280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,753 DEBUG [Thread-1295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,754 DEBUG [Thread-1282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,761 DEBUG [Thread-1291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,761 DEBUG [Thread-1291 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,762 DEBUG [Thread-1291 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,768 DEBUG [Thread-1280 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,769 DEBUG [Thread-1280 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,787 DEBUG [Thread-1299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:15:31,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:15:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:15:31,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:15:31,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:15:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:31,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T17:15:31,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/25dfb884632448d7b76d41425b3895e3 is 50, key is test_row_0/A:col10/1733159731789/Put/seqid=0 2024-12-02T17:15:31,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159791853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159791855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159791859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159791862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159791865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T17:15:31,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:31,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:31,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:31,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:31,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742105_1281 (size=12001) 2024-12-02T17:15:31,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/25dfb884632448d7b76d41425b3895e3 2024-12-02T17:15:31,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/c3e2c85f428545489263bd13f1300329 is 50, key is test_row_0/B:col10/1733159731789/Put/seqid=0 2024-12-02T17:15:31,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159791967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159791976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159791976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159791987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:31,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159791987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742106_1282 (size=12001) 2024-12-02T17:15:32,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/c3e2c85f428545489263bd13f1300329 2024-12-02T17:15:32,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T17:15:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T17:15:32,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/82f4086176a64a83958e1903c6605071 is 50, key is test_row_0/C:col10/1733159731789/Put/seqid=0 2024-12-02T17:15:32,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742107_1283 (size=12001) 2024-12-02T17:15:32,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/82f4086176a64a83958e1903c6605071 2024-12-02T17:15:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/25dfb884632448d7b76d41425b3895e3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/25dfb884632448d7b76d41425b3895e3 2024-12-02T17:15:32,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/25dfb884632448d7b76d41425b3895e3, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T17:15:32,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/c3e2c85f428545489263bd13f1300329 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c3e2c85f428545489263bd13f1300329 2024-12-02T17:15:32,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c3e2c85f428545489263bd13f1300329, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T17:15:32,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/82f4086176a64a83958e1903c6605071 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/82f4086176a64a83958e1903c6605071 2024-12-02T17:15:32,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/82f4086176a64a83958e1903c6605071, entries=150, sequenceid=13, filesize=11.7 K 2024-12-02T17:15:32,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cfac2600ff5eef41b88bb4f556e87599 in 338ms, sequenceid=13, compaction requested=false 2024-12-02T17:15:32,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:32,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T17:15:32,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:32,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:32,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:32,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/7698952280ab46a7a7d20889d2a13fe1 is 50, key is test_row_0/A:col10/1733159731846/Put/seqid=0 2024-12-02T17:15:32,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T17:15:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159792215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159792215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159792218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159792218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742108_1284 (size=14341) 2024-12-02T17:15:32,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159792225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/7698952280ab46a7a7d20889d2a13fe1 2024-12-02T17:15:32,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/fa6171d9cb5740f0874e8003dab2687d is 50, key is test_row_0/B:col10/1733159731846/Put/seqid=0 2024-12-02T17:15:32,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742109_1285 (size=12001) 2024-12-02T17:15:32,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/fa6171d9cb5740f0874e8003dab2687d 2024-12-02T17:15:32,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/ce2b8e744dfe4b2ebdf17325e4462447 is 50, key is test_row_0/C:col10/1733159731846/Put/seqid=0 2024-12-02T17:15:32,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159792337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159792337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159792337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159792337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159792337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T17:15:32,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T17:15:32,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:32,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:32,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742110_1286 (size=12001) 2024-12-02T17:15:32,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/ce2b8e744dfe4b2ebdf17325e4462447 2024-12-02T17:15:32,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/7698952280ab46a7a7d20889d2a13fe1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/7698952280ab46a7a7d20889d2a13fe1 2024-12-02T17:15:32,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/7698952280ab46a7a7d20889d2a13fe1, entries=200, sequenceid=34, filesize=14.0 K 2024-12-02T17:15:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/fa6171d9cb5740f0874e8003dab2687d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/fa6171d9cb5740f0874e8003dab2687d 2024-12-02T17:15:32,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/fa6171d9cb5740f0874e8003dab2687d, entries=150, sequenceid=34, filesize=11.7 K 2024-12-02T17:15:32,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/ce2b8e744dfe4b2ebdf17325e4462447 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/ce2b8e744dfe4b2ebdf17325e4462447 2024-12-02T17:15:32,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/ce2b8e744dfe4b2ebdf17325e4462447, entries=150, sequenceid=34, filesize=11.7 K 2024-12-02T17:15:32,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for cfac2600ff5eef41b88bb4f556e87599 in 220ms, sequenceid=34, compaction requested=false 2024-12-02T17:15:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:32,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-02T17:15:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:32,542 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:32,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:32,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/d7a57066418c48fb80d5091854a4ef17 is 50, key is test_row_0/A:col10/1733159732213/Put/seqid=0 2024-12-02T17:15:32,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742111_1287 (size=12001) 2024-12-02T17:15:32,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159792596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159732588, max timestamp=1733159732606Num uniques 859; 2024-12-02T17:15:32,610 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/d7a57066418c48fb80d5091854a4ef17 2024-12-02T17:15:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159792597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159732588, max timestamp=1733159732606Num uniques 859; 2024-12-02T17:15:32,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:15:32,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159732588, max timestamp=1733159732606Num uniques 859; 2024-12-02T17:15:32,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:15:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/27fbce7191764784a944b8d896668e1d is 50, key is test_row_0/B:col10/1733159732213/Put/seqid=0 2024-12-02T17:15:32,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742112_1288 (size=12001) 2024-12-02T17:15:32,658 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/27fbce7191764784a944b8d896668e1d 2024-12-02T17:15:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/8959a3680baf4b7e9aff6cb429cb435f is 50, key is test_row_0/C:col10/1733159732213/Put/seqid=0 2024-12-02T17:15:32,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159792679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159792682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159792689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742113_1289 (size=12001) 2024-12-02T17:15:32,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159792717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159792717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159792797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159792798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159792806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T17:15:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159792937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:32,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159792958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159793021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159793021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159793027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,132 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/8959a3680baf4b7e9aff6cb429cb435f 2024-12-02T17:15:33,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/d7a57066418c48fb80d5091854a4ef17 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d7a57066418c48fb80d5091854a4ef17 2024-12-02T17:15:33,159 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d7a57066418c48fb80d5091854a4ef17, entries=150, sequenceid=45, filesize=11.7 K 2024-12-02T17:15:33,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/27fbce7191764784a944b8d896668e1d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/27fbce7191764784a944b8d896668e1d 2024-12-02T17:15:33,175 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/27fbce7191764784a944b8d896668e1d, entries=150, sequenceid=45, filesize=11.7 K 2024-12-02T17:15:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/8959a3680baf4b7e9aff6cb429cb435f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/8959a3680baf4b7e9aff6cb429cb435f 2024-12-02T17:15:33,185 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/8959a3680baf4b7e9aff6cb429cb435f, entries=150, sequenceid=45, filesize=11.7 K 2024-12-02T17:15:33,186 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for cfac2600ff5eef41b88bb4f556e87599 in 644ms, sequenceid=45, compaction requested=true 2024-12-02T17:15:33,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:33,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:33,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-02T17:15:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-02T17:15:33,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-02T17:15:33,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-12-02T17:15:33,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.4580 sec 2024-12-02T17:15:33,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-12-02T17:15:33,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:33,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:33,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:33,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:33,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:33,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:33,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/9718de60f1df4bc7876135b96f27794d is 50, key is test_row_0/A:col10/1733159732675/Put/seqid=0 2024-12-02T17:15:33,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742114_1290 (size=14341) 2024-12-02T17:15:33,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159793327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159793329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159793337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159793347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159793378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159793447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159793447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159793657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159793667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/9718de60f1df4bc7876135b96f27794d 2024-12-02T17:15:33,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/9611188c0a304f3cb98418a80f7002a2 is 50, key is test_row_0/B:col10/1733159732675/Put/seqid=0 2024-12-02T17:15:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742115_1291 (size=12001) 2024-12-02T17:15:33,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/9611188c0a304f3cb98418a80f7002a2 2024-12-02T17:15:33,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/69e8617604a84aa29999d770fd9cb3ce is 50, key is test_row_0/C:col10/1733159732675/Put/seqid=0 2024-12-02T17:15:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742116_1292 (size=12001) 2024-12-02T17:15:33,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/69e8617604a84aa29999d770fd9cb3ce 2024-12-02T17:15:33,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/9718de60f1df4bc7876135b96f27794d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/9718de60f1df4bc7876135b96f27794d 2024-12-02T17:15:33,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/9718de60f1df4bc7876135b96f27794d, entries=200, sequenceid=82, filesize=14.0 K 2024-12-02T17:15:33,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/9611188c0a304f3cb98418a80f7002a2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9611188c0a304f3cb98418a80f7002a2 2024-12-02T17:15:33,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159793856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159793867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9611188c0a304f3cb98418a80f7002a2, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T17:15:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-02T17:15:33,889 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:33,890 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-02T17:15:33,892 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:33,893 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:33,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T17:15:33,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/69e8617604a84aa29999d770fd9cb3ce as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/69e8617604a84aa29999d770fd9cb3ce 2024-12-02T17:15:33,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159793899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:33,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/69e8617604a84aa29999d770fd9cb3ce, entries=150, sequenceid=82, filesize=11.7 K 2024-12-02T17:15:33,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.39 KB/136590, heapSize ~383.11 KB/392304, currentSize=46.96 KB/48090 for cfac2600ff5eef41b88bb4f556e87599 in 666ms, sequenceid=82, compaction requested=true 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:33,920 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:33,920 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:33,922 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:33,922 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:33,922 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:33,922 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:33,922 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:33,922 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c3e2c85f428545489263bd13f1300329, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/fa6171d9cb5740f0874e8003dab2687d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/27fbce7191764784a944b8d896668e1d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9611188c0a304f3cb98418a80f7002a2] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=46.9 K 2024-12-02T17:15:33,922 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:33,922 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/25dfb884632448d7b76d41425b3895e3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/7698952280ab46a7a7d20889d2a13fe1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d7a57066418c48fb80d5091854a4ef17, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/9718de60f1df4bc7876135b96f27794d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=51.4 K 2024-12-02T17:15:33,923 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e2c85f428545489263bd13f1300329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733159731766 2024-12-02T17:15:33,923 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25dfb884632448d7b76d41425b3895e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733159731766 2024-12-02T17:15:33,923 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fa6171d9cb5740f0874e8003dab2687d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733159731846 2024-12-02T17:15:33,923 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7698952280ab46a7a7d20889d2a13fe1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733159731846 2024-12-02T17:15:33,923 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fbce7191764784a944b8d896668e1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733159732203 2024-12-02T17:15:33,924 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7a57066418c48fb80d5091854a4ef17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733159732203 2024-12-02T17:15:33,924 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9611188c0a304f3cb98418a80f7002a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733159732675 2024-12-02T17:15:33,924 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9718de60f1df4bc7876135b96f27794d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733159732666 2024-12-02T17:15:33,949 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#243 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:33,949 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/106420c9deb0442fafecc8c804bd7d74 is 50, key is test_row_0/B:col10/1733159732675/Put/seqid=0 2024-12-02T17:15:33,960 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:33,965 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/1beb36860dd047a3be96bddfcf87e2c7 is 50, key is test_row_0/A:col10/1733159732675/Put/seqid=0 2024-12-02T17:15:33,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T17:15:34,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742117_1293 (size=12139) 2024-12-02T17:15:34,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/6504d26915c14c22abcc2589b2583e5d is 50, key is test_row_0/A:col10/1733159733284/Put/seqid=0 2024-12-02T17:15:34,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T17:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,055 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/106420c9deb0442fafecc8c804bd7d74 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/106420c9deb0442fafecc8c804bd7d74 2024-12-02T17:15:34,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742118_1294 (size=12139) 2024-12-02T17:15:34,070 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/1beb36860dd047a3be96bddfcf87e2c7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1beb36860dd047a3be96bddfcf87e2c7 2024-12-02T17:15:34,070 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 106420c9deb0442fafecc8c804bd7d74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:34,070 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:34,071 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=12, startTime=1733159733920; duration=0sec 2024-12-02T17:15:34,071 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:34,071 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:34,071 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:34,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:34,073 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:34,073 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,073 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/82f4086176a64a83958e1903c6605071, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/ce2b8e744dfe4b2ebdf17325e4462447, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/8959a3680baf4b7e9aff6cb429cb435f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/69e8617604a84aa29999d770fd9cb3ce] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=46.9 K 2024-12-02T17:15:34,079 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f4086176a64a83958e1903c6605071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733159731766 2024-12-02T17:15:34,080 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2b8e744dfe4b2ebdf17325e4462447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733159731846 2024-12-02T17:15:34,080 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8959a3680baf4b7e9aff6cb429cb435f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733159732203 2024-12-02T17:15:34,080 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 69e8617604a84aa29999d770fd9cb3ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733159732675 2024-12-02T17:15:34,109 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:34,109 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d0479bc13ab741d7b3504e8afe6d0f82 is 50, key is test_row_0/C:col10/1733159732675/Put/seqid=0 2024-12-02T17:15:34,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742119_1295 (size=11997) 2024-12-02T17:15:34,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/6504d26915c14c22abcc2589b2583e5d 2024-12-02T17:15:34,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/47d2490a890a4156b20dd25c6843965f is 50, key is test_row_0/B:col10/1733159733284/Put/seqid=0 2024-12-02T17:15:34,136 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 1beb36860dd047a3be96bddfcf87e2c7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:34,136 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:34,136 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=12, startTime=1733159733920; duration=0sec 2024-12-02T17:15:34,136 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:34,136 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742120_1296 (size=12139) 2024-12-02T17:15:34,160 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d0479bc13ab741d7b3504e8afe6d0f82 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d0479bc13ab741d7b3504e8afe6d0f82 2024-12-02T17:15:34,168 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into d0479bc13ab741d7b3504e8afe6d0f82(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:34,168 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:34,168 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=12, startTime=1733159733920; duration=0sec 2024-12-02T17:15:34,168 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:34,168 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:34,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159794171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159734155, max timestamp=1733159734169Num uniques 859; 2024-12-02T17:15:34,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159734155, max timestamp=1733159734169Num uniques 859; 2024-12-02T17:15:34,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:34,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159734155, max timestamp=1733159734169Num uniques 859; 2024-12-02T17:15:34,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:34,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742121_1297 (size=9657) 2024-12-02T17:15:34,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/47d2490a890a4156b20dd25c6843965f 2024-12-02T17:15:34,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:34,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/041ab73ba5434866967f6772917ae71e is 50, key is test_row_0/C:col10/1733159733284/Put/seqid=0 2024-12-02T17:15:34,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T17:15:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T17:15:34,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742122_1298 (size=9657) 2024-12-02T17:15:34,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159794297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159794298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T17:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159794407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159794416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T17:15:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T17:15:34,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159794617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159794626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/041ab73ba5434866967f6772917ae71e 2024-12-02T17:15:34,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/6504d26915c14c22abcc2589b2583e5d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6504d26915c14c22abcc2589b2583e5d 2024-12-02T17:15:34,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6504d26915c14c22abcc2589b2583e5d, entries=150, sequenceid=93, filesize=11.7 K 2024-12-02T17:15:34,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/47d2490a890a4156b20dd25c6843965f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/47d2490a890a4156b20dd25c6843965f 2024-12-02T17:15:34,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/47d2490a890a4156b20dd25c6843965f, entries=100, sequenceid=93, filesize=9.4 K 2024-12-02T17:15:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/041ab73ba5434866967f6772917ae71e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/041ab73ba5434866967f6772917ae71e 2024-12-02T17:15:34,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T17:15:34,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:34,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:34,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/041ab73ba5434866967f6772917ae71e, entries=100, sequenceid=93, filesize=9.4 K 2024-12-02T17:15:34,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cfac2600ff5eef41b88bb4f556e87599 in 722ms, sequenceid=93, compaction requested=false 2024-12-02T17:15:34,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:34,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:34,833 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:34,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/b56be8a4444e4c6bae7226357a2c77ea is 50, key is test_row_0/A:col10/1733159734275/Put/seqid=0 2024-12-02T17:15:34,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742123_1299 (size=12151) 2024-12-02T17:15:34,862 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/b56be8a4444e4c6bae7226357a2c77ea 2024-12-02T17:15:34,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/250111df010c40939ba2bd895145913b is 50, key is test_row_0/B:col10/1733159734275/Put/seqid=0 2024-12-02T17:15:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:34,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:34,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742124_1300 (size=12151) 2024-12-02T17:15:34,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159794917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159794917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,923 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159794921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159794929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:34,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159794937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:34,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:34,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159795027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T17:15:35,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159795047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159795238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159795257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,283 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/250111df010c40939ba2bd895145913b 2024-12-02T17:15:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/6e9200bb7a5f44059e1f04fe300c97a1 is 50, key is test_row_0/C:col10/1733159734275/Put/seqid=0 2024-12-02T17:15:35,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742125_1301 (size=12151) 2024-12-02T17:15:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159795447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159795448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159795547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159795567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:35,741 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/6e9200bb7a5f44059e1f04fe300c97a1 2024-12-02T17:15:35,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/b56be8a4444e4c6bae7226357a2c77ea as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/b56be8a4444e4c6bae7226357a2c77ea 2024-12-02T17:15:35,761 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/b56be8a4444e4c6bae7226357a2c77ea, entries=150, sequenceid=133, filesize=11.9 K 2024-12-02T17:15:35,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/250111df010c40939ba2bd895145913b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/250111df010c40939ba2bd895145913b 2024-12-02T17:15:35,775 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/250111df010c40939ba2bd895145913b, entries=150, sequenceid=133, filesize=11.9 K 2024-12-02T17:15:35,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/6e9200bb7a5f44059e1f04fe300c97a1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/6e9200bb7a5f44059e1f04fe300c97a1 2024-12-02T17:15:35,782 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/6e9200bb7a5f44059e1f04fe300c97a1, entries=150, sequenceid=133, filesize=11.9 K 2024-12-02T17:15:35,783 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cfac2600ff5eef41b88bb4f556e87599 in 950ms, sequenceid=133, compaction requested=true 2024-12-02T17:15:35,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:35,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:35,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-02T17:15:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-02T17:15:35,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-02T17:15:35,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-12-02T17:15:35,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8980 sec 2024-12-02T17:15:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-02T17:15:36,048 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:36,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-02T17:15:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T17:15:36,052 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:36,052 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:36,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:36,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:36,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:36,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/1e5a331762ab49d5ba3462bd2e6e7b75 is 50, key is test_row_0/A:col10/1733159734891/Put/seqid=0 2024-12-02T17:15:36,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742126_1302 (size=14541) 2024-12-02T17:15:36,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159736111, max timestamp=1733159736142Num uniques 859; 2024-12-02T17:15:36,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:36,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159736111, max timestamp=1733159736142Num uniques 859; 2024-12-02T17:15:36,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:36,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159736111, max timestamp=1733159736142Num uniques 859; 2024-12-02T17:15:36,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T17:15:36,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T17:15:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159796253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159796262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,285 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:15:36,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T17:15:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T17:15:36,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159796367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159796378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159796467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159796467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,474 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,474 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/1e5a331762ab49d5ba3462bd2e6e7b75 2024-12-02T17:15:36,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/f6b64bfde5ee4c8785f871dedf4a9c86 is 50, key is test_row_0/B:col10/1733159734891/Put/seqid=0 2024-12-02T17:15:36,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T17:15:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:36,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742127_1303 (size=12151) 2024-12-02T17:15:36,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/f6b64bfde5ee4c8785f871dedf4a9c86 2024-12-02T17:15:36,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d8706968490d49669bb303bc08d7aaa8 is 50, key is test_row_0/C:col10/1733159734891/Put/seqid=0 2024-12-02T17:15:36,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742128_1304 (size=12151) 2024-12-02T17:15:36,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d8706968490d49669bb303bc08d7aaa8 2024-12-02T17:15:36,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159796587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159796587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/1e5a331762ab49d5ba3462bd2e6e7b75 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1e5a331762ab49d5ba3462bd2e6e7b75 2024-12-02T17:15:36,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1e5a331762ab49d5ba3462bd2e6e7b75, entries=200, sequenceid=144, filesize=14.2 K 2024-12-02T17:15:36,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/f6b64bfde5ee4c8785f871dedf4a9c86 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f6b64bfde5ee4c8785f871dedf4a9c86 2024-12-02T17:15:36,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f6b64bfde5ee4c8785f871dedf4a9c86, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T17:15:36,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d8706968490d49669bb303bc08d7aaa8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d8706968490d49669bb303bc08d7aaa8 2024-12-02T17:15:36,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d8706968490d49669bb303bc08d7aaa8, entries=150, sequenceid=144, filesize=11.9 K 2024-12-02T17:15:36,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cfac2600ff5eef41b88bb4f556e87599 in 552ms, sequenceid=144, compaction requested=true 2024-12-02T17:15:36,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:36,614 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:36,614 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:36,616 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50828 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:36,616 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:36,616 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,616 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1beb36860dd047a3be96bddfcf87e2c7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6504d26915c14c22abcc2589b2583e5d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/b56be8a4444e4c6bae7226357a2c77ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1e5a331762ab49d5ba3462bd2e6e7b75] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=49.6 K 2024-12-02T17:15:36,616 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:36,617 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:36,617 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,617 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/106420c9deb0442fafecc8c804bd7d74, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/47d2490a890a4156b20dd25c6843965f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/250111df010c40939ba2bd895145913b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f6b64bfde5ee4c8785f871dedf4a9c86] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=45.0 K 2024-12-02T17:15:36,617 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106420c9deb0442fafecc8c804bd7d74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733159732675 2024-12-02T17:15:36,617 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1beb36860dd047a3be96bddfcf87e2c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733159732675 2024-12-02T17:15:36,618 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6504d26915c14c22abcc2589b2583e5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733159733284 2024-12-02T17:15:36,618 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d2490a890a4156b20dd25c6843965f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733159733284 2024-12-02T17:15:36,618 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b56be8a4444e4c6bae7226357a2c77ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733159734275 2024-12-02T17:15:36,619 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 250111df010c40939ba2bd895145913b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733159734275 2024-12-02T17:15:36,619 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5a331762ab49d5ba3462bd2e6e7b75, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733159734891 2024-12-02T17:15:36,620 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6b64bfde5ee4c8785f871dedf4a9c86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733159734891 2024-12-02T17:15:36,649 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:36,650 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/f8f151132dd14be987948ddc188e11da is 50, key is test_row_0/B:col10/1733159734891/Put/seqid=0 2024-12-02T17:15:36,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,673 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:36,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T17:15:36,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742129_1305 (size=12425) 2024-12-02T17:15:36,679 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#256 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:36,679 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/041d7abbe2664b6683a30e738f014832 is 50, key is test_row_0/A:col10/1733159734891/Put/seqid=0 2024-12-02T17:15:36,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/66a5bd51208b44ba969fd081e1537f66 is 50, key is test_row_0/A:col10/1733159736203/Put/seqid=0 2024-12-02T17:15:36,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742130_1306 (size=12425) 2024-12-02T17:15:36,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742131_1307 (size=12151) 2024-12-02T17:15:36,752 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/041d7abbe2664b6683a30e738f014832 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/041d7abbe2664b6683a30e738f014832 2024-12-02T17:15:36,752 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/66a5bd51208b44ba969fd081e1537f66 2024-12-02T17:15:36,759 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 041d7abbe2664b6683a30e738f014832(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:36,759 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:36,760 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=12, startTime=1733159736614; duration=0sec 2024-12-02T17:15:36,760 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:36,760 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:36,760 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:36,761 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46098 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:36,761 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:36,762 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:36,762 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d0479bc13ab741d7b3504e8afe6d0f82, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/041ab73ba5434866967f6772917ae71e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/6e9200bb7a5f44059e1f04fe300c97a1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d8706968490d49669bb303bc08d7aaa8] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=45.0 K 2024-12-02T17:15:36,762 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d0479bc13ab741d7b3504e8afe6d0f82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733159732675 2024-12-02T17:15:36,762 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 041ab73ba5434866967f6772917ae71e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733159733284 2024-12-02T17:15:36,763 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9200bb7a5f44059e1f04fe300c97a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733159734275 2024-12-02T17:15:36,763 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d8706968490d49669bb303bc08d7aaa8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733159734891 2024-12-02T17:15:36,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/7c3fa51b2ea84226b7f1ee3481532584 is 50, key is test_row_0/B:col10/1733159736203/Put/seqid=0 2024-12-02T17:15:36,787 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:36,787 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/83019e4ef87e48efa583a5d3b77249c8 is 50, key is test_row_0/C:col10/1733159734891/Put/seqid=0 2024-12-02T17:15:36,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742132_1308 (size=12151) 2024-12-02T17:15:36,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742133_1309 (size=12425) 2024-12-02T17:15:36,865 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/83019e4ef87e48efa583a5d3b77249c8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83019e4ef87e48efa583a5d3b77249c8 2024-12-02T17:15:36,873 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 83019e4ef87e48efa583a5d3b77249c8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:36,873 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:36,873 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=12, startTime=1733159736614; duration=0sec 2024-12-02T17:15:36,874 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:36,874 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:36,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159796925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159796933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159796937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:36,943 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159797038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:37,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159797047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,086 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/f8f151132dd14be987948ddc188e11da as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f8f151132dd14be987948ddc188e11da 2024-12-02T17:15:37,093 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into f8f151132dd14be987948ddc188e11da(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:37,093 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:37,093 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=12, startTime=1733159736614; duration=0sec 2024-12-02T17:15:37,093 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:37,093 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T17:15:37,239 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/7c3fa51b2ea84226b7f1ee3481532584 2024-12-02T17:15:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159797248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159797258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b5ea2ec327a64be9b6ab4b260d4f022d is 50, key is test_row_0/C:col10/1733159736203/Put/seqid=0 2024-12-02T17:15:37,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742134_1310 (size=12151) 2024-12-02T17:15:37,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159797568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:37,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159797576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:37,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:37,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b5ea2ec327a64be9b6ab4b260d4f022d 2024-12-02T17:15:37,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/66a5bd51208b44ba969fd081e1537f66 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/66a5bd51208b44ba969fd081e1537f66 2024-12-02T17:15:37,769 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/66a5bd51208b44ba969fd081e1537f66, entries=150, sequenceid=181, filesize=11.9 K 2024-12-02T17:15:37,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/7c3fa51b2ea84226b7f1ee3481532584 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7c3fa51b2ea84226b7f1ee3481532584 2024-12-02T17:15:37,776 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7c3fa51b2ea84226b7f1ee3481532584, entries=150, sequenceid=181, filesize=11.9 K 2024-12-02T17:15:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b5ea2ec327a64be9b6ab4b260d4f022d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b5ea2ec327a64be9b6ab4b260d4f022d 2024-12-02T17:15:37,781 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b5ea2ec327a64be9b6ab4b260d4f022d, entries=150, sequenceid=181, filesize=11.9 K 2024-12-02T17:15:37,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cfac2600ff5eef41b88bb4f556e87599 in 1110ms, sequenceid=181, compaction requested=false 2024-12-02T17:15:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-02T17:15:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-02T17:15:37,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-02T17:15:37,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-12-02T17:15:37,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7410 sec 2024-12-02T17:15:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:38,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:38,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:38,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2b5ce353710b499d85c0c23747a80a2f is 50, key is test_row_0/A:col10/1733159736922/Put/seqid=0 2024-12-02T17:15:38,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742135_1311 (size=14541) 2024-12-02T17:15:38,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159738125, max timestamp=1733159738141Num uniques 859; 2024-12-02T17:15:38,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:38,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159738125, max timestamp=1733159738141Num uniques 859; 2024-12-02T17:15:38,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:38,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159738125, max timestamp=1733159738141Num uniques 859; 2024-12-02T17:15:38,147 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T17:15:38,197 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:38,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-02T17:15:38,200 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T17:15:38,201 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:38,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:38,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159798213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159798213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T17:15:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159798327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159798329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T17:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159798479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,480 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2b5ce353710b499d85c0c23747a80a2f 2024-12-02T17:15:38,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159798497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,502 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T17:15:38,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T17:15:38,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8245698f24a14d46bf8e62e51f4b677a is 50, key is test_row_0/B:col10/1733159736922/Put/seqid=0 2024-12-02T17:15:38,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742136_1312 (size=12151) 2024-12-02T17:15:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159798536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159798537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8245698f24a14d46bf8e62e51f4b677a 2024-12-02T17:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/74b21c110e6941a6baf33572328639a3 is 50, key is test_row_0/C:col10/1733159736922/Put/seqid=0 2024-12-02T17:15:38,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742137_1313 (size=12151) 2024-12-02T17:15:38,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T17:15:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T17:15:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:38,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T17:15:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159798847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159798847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:38,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/74b21c110e6941a6baf33572328639a3 2024-12-02T17:15:38,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T17:15:38,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:38,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:38,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2b5ce353710b499d85c0c23747a80a2f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2b5ce353710b499d85c0c23747a80a2f 2024-12-02T17:15:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:38,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2b5ce353710b499d85c0c23747a80a2f, entries=200, sequenceid=195, filesize=14.2 K 2024-12-02T17:15:38,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8245698f24a14d46bf8e62e51f4b677a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8245698f24a14d46bf8e62e51f4b677a 2024-12-02T17:15:38,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8245698f24a14d46bf8e62e51f4b677a, entries=150, sequenceid=195, filesize=11.9 K 2024-12-02T17:15:38,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/74b21c110e6941a6baf33572328639a3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/74b21c110e6941a6baf33572328639a3 2024-12-02T17:15:39,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/74b21c110e6941a6baf33572328639a3, entries=150, sequenceid=195, filesize=11.9 K 2024-12-02T17:15:39,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cfac2600ff5eef41b88bb4f556e87599 in 927ms, sequenceid=195, compaction requested=true 2024-12-02T17:15:39,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:39,006 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:39,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:39,007 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:39,007 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:39,007 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:39,007 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/041d7abbe2664b6683a30e738f014832, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/66a5bd51208b44ba969fd081e1537f66, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2b5ce353710b499d85c0c23747a80a2f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=38.2 K 2024-12-02T17:15:39,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:39,008 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:39,008 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 041d7abbe2664b6683a30e738f014832, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733159734891 2024-12-02T17:15:39,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:39,009 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a5bd51208b44ba969fd081e1537f66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733159736203 2024-12-02T17:15:39,009 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:39,009 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:39,009 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:39,009 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f8f151132dd14be987948ddc188e11da, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7c3fa51b2ea84226b7f1ee3481532584, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8245698f24a14d46bf8e62e51f4b677a] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=35.9 K 2024-12-02T17:15:39,010 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5ce353710b499d85c0c23747a80a2f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733159736922 2024-12-02T17:15:39,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting f8f151132dd14be987948ddc188e11da, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733159734891 2024-12-02T17:15:39,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:39,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:39,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3fa51b2ea84226b7f1ee3481532584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733159736203 2024-12-02T17:15:39,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:39,011 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8245698f24a14d46bf8e62e51f4b677a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733159736922 2024-12-02T17:15:39,027 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:39,028 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/4684d4975f2641ea9df076ccc1eae0a3 is 50, key is test_row_0/A:col10/1733159736922/Put/seqid=0 2024-12-02T17:15:39,036 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:39,038 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/c76399da9e3a49b18197184746ee9817 is 50, key is test_row_0/B:col10/1733159736922/Put/seqid=0 2024-12-02T17:15:39,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742138_1314 (size=12527) 2024-12-02T17:15:39,084 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/4684d4975f2641ea9df076ccc1eae0a3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/4684d4975f2641ea9df076ccc1eae0a3 2024-12-02T17:15:39,092 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 4684d4975f2641ea9df076ccc1eae0a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:39,092 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:39,092 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159739006; duration=0sec 2024-12-02T17:15:39,092 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:39,092 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:39,092 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:39,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742139_1315 (size=12527) 2024-12-02T17:15:39,095 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:39,097 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:39,097 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:39,097 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83019e4ef87e48efa583a5d3b77249c8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b5ea2ec327a64be9b6ab4b260d4f022d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/74b21c110e6941a6baf33572328639a3] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=35.9 K 2024-12-02T17:15:39,099 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83019e4ef87e48efa583a5d3b77249c8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733159734891 2024-12-02T17:15:39,099 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5ea2ec327a64be9b6ab4b260d4f022d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733159736203 2024-12-02T17:15:39,100 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/c76399da9e3a49b18197184746ee9817 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c76399da9e3a49b18197184746ee9817 2024-12-02T17:15:39,106 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b21c110e6941a6baf33572328639a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733159736922 2024-12-02T17:15:39,116 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into c76399da9e3a49b18197184746ee9817(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:39,116 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:39,118 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159739008; duration=0sec 2024-12-02T17:15:39,118 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:39,118 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:39,124 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:39,125 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/981969d8a6214805866c44e8bd290ed0 is 50, key is test_row_0/C:col10/1733159736922/Put/seqid=0 2024-12-02T17:15:39,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:39,135 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:39,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742140_1316 (size=12527) 2024-12-02T17:15:39,176 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/981969d8a6214805866c44e8bd290ed0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/981969d8a6214805866c44e8bd290ed0 2024-12-02T17:15:39,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/61ff89374b484414bbc0f1cc1d3bacc4 is 50, key is test_row_0/A:col10/1733159738211/Put/seqid=0 2024-12-02T17:15:39,186 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 981969d8a6214805866c44e8bd290ed0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:39,186 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:39,186 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159739010; duration=0sec 2024-12-02T17:15:39,186 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:39,186 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742141_1317 (size=12151) 2024-12-02T17:15:39,205 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/61ff89374b484414bbc0f1cc1d3bacc4 2024-12-02T17:15:39,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/bc92f0305ad74a60bfd6d5b5725e287d is 50, key is test_row_0/B:col10/1733159738211/Put/seqid=0 2024-12-02T17:15:39,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742142_1318 (size=12151) 2024-12-02T17:15:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T17:15:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:39,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:39,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159799400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:39,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159799404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159799508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159799517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:39,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/bc92f0305ad74a60bfd6d5b5725e287d 2024-12-02T17:15:39,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/95ddbd443cde4f92ac02e1d5ef99f632 is 50, key is test_row_0/C:col10/1733159738211/Put/seqid=0 2024-12-02T17:15:39,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742143_1319 (size=12151) 2024-12-02T17:15:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/95ddbd443cde4f92ac02e1d5ef99f632 2024-12-02T17:15:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/61ff89374b484414bbc0f1cc1d3bacc4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/61ff89374b484414bbc0f1cc1d3bacc4 2024-12-02T17:15:39,714 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/61ff89374b484414bbc0f1cc1d3bacc4, entries=150, sequenceid=233, filesize=11.9 K 2024-12-02T17:15:39,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/bc92f0305ad74a60bfd6d5b5725e287d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/bc92f0305ad74a60bfd6d5b5725e287d 2024-12-02T17:15:39,720 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/bc92f0305ad74a60bfd6d5b5725e287d, entries=150, sequenceid=233, filesize=11.9 K 2024-12-02T17:15:39,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/95ddbd443cde4f92ac02e1d5ef99f632 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95ddbd443cde4f92ac02e1d5ef99f632 2024-12-02T17:15:39,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159799725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:39,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159799727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:39,744 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95ddbd443cde4f92ac02e1d5ef99f632, entries=150, sequenceid=233, filesize=11.9 K 2024-12-02T17:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:39,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:39,746 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for cfac2600ff5eef41b88bb4f556e87599 in 611ms, sequenceid=233, compaction requested=false 2024-12-02T17:15:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-02T17:15:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-02T17:15:39,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-02T17:15:39,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5470 sec 2024-12-02T17:15:39,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5530 sec 2024-12-02T17:15:40,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:40,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:40,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:40,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:40,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/29b154a9a63140adac7555c742aa138f is 50, key is test_row_0/A:col10/1733159739395/Put/seqid=0 2024-12-02T17:15:40,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742144_1320 (size=14541) 2024-12-02T17:15:40,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/29b154a9a63140adac7555c742aa138f 2024-12-02T17:15:40,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/ab98336b750b417a864cbbb795c02d40 is 50, key is test_row_0/B:col10/1733159739395/Put/seqid=0 2024-12-02T17:15:40,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159740086, max timestamp=1733159740102Num uniques 859; 2024-12-02T17:15:40,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:40,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159740086, max timestamp=1733159740102Num uniques 859; 2024-12-02T17:15:40,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:40,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159740086, max timestamp=1733159740102Num uniques 859; 2024-12-02T17:15:40,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:40,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742145_1321 (size=12151) 2024-12-02T17:15:40,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/ab98336b750b417a864cbbb795c02d40 2024-12-02T17:15:40,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/609eea962027457881921c4c95c5fdb5 is 50, key is test_row_0/C:col10/1733159739395/Put/seqid=0 2024-12-02T17:15:40,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742146_1322 (size=12151) 2024-12-02T17:15:40,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/609eea962027457881921c4c95c5fdb5 2024-12-02T17:15:40,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/29b154a9a63140adac7555c742aa138f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/29b154a9a63140adac7555c742aa138f 2024-12-02T17:15:40,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/29b154a9a63140adac7555c742aa138f, entries=200, sequenceid=245, filesize=14.2 K 2024-12-02T17:15:40,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/ab98336b750b417a864cbbb795c02d40 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/ab98336b750b417a864cbbb795c02d40 2024-12-02T17:15:40,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/ab98336b750b417a864cbbb795c02d40, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T17:15:40,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/609eea962027457881921c4c95c5fdb5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/609eea962027457881921c4c95c5fdb5 2024-12-02T17:15:40,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/609eea962027457881921c4c95c5fdb5, entries=150, sequenceid=245, filesize=11.9 K 2024-12-02T17:15:40,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cfac2600ff5eef41b88bb4f556e87599 in 140ms, sequenceid=245, compaction requested=true 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:40,180 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:40,180 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:40,181 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:40,181 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:40,181 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:40,181 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:40,181 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,181 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,181 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c76399da9e3a49b18197184746ee9817, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/bc92f0305ad74a60bfd6d5b5725e287d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/ab98336b750b417a864cbbb795c02d40] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.0 K 2024-12-02T17:15:40,181 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/4684d4975f2641ea9df076ccc1eae0a3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/61ff89374b484414bbc0f1cc1d3bacc4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/29b154a9a63140adac7555c742aa138f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=38.3 K 2024-12-02T17:15:40,182 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4684d4975f2641ea9df076ccc1eae0a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733159736922 2024-12-02T17:15:40,182 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c76399da9e3a49b18197184746ee9817, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733159736922 2024-12-02T17:15:40,182 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting bc92f0305ad74a60bfd6d5b5725e287d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733159738186 2024-12-02T17:15:40,182 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ff89374b484414bbc0f1cc1d3bacc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733159738186 2024-12-02T17:15:40,182 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ab98336b750b417a864cbbb795c02d40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159739394 2024-12-02T17:15:40,183 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b154a9a63140adac7555c742aa138f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159739394 2024-12-02T17:15:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:40,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-02T17:15:40,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:40,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:40,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f21fabb90c9041b3979383fdbea0fcfa is 50, key is test_row_0/A:col10/1733159740182/Put/seqid=0 2024-12-02T17:15:40,211 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:40,211 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/e7a6442c3f7a42d5abbf687908216dfc is 50, key is test_row_0/B:col10/1733159739395/Put/seqid=0 2024-12-02T17:15:40,214 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:40,215 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/8dab7d824fcf4631b7b7ab6ebc201213 is 50, key is test_row_0/A:col10/1733159739395/Put/seqid=0 2024-12-02T17:15:40,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159800213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159800233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742149_1325 (size=12629) 2024-12-02T17:15:40,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742148_1324 (size=12629) 2024-12-02T17:15:40,273 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/8dab7d824fcf4631b7b7ab6ebc201213 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8dab7d824fcf4631b7b7ab6ebc201213 2024-12-02T17:15:40,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742147_1323 (size=12301) 2024-12-02T17:15:40,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f21fabb90c9041b3979383fdbea0fcfa 2024-12-02T17:15:40,282 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 8dab7d824fcf4631b7b7ab6ebc201213(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:40,282 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:40,282 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159740180; duration=0sec 2024-12-02T17:15:40,282 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:40,282 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:40,282 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:40,288 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:40,288 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:40,288 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,288 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/981969d8a6214805866c44e8bd290ed0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95ddbd443cde4f92ac02e1d5ef99f632, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/609eea962027457881921c4c95c5fdb5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.0 K 2024-12-02T17:15:40,289 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981969d8a6214805866c44e8bd290ed0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1733159736922 2024-12-02T17:15:40,289 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95ddbd443cde4f92ac02e1d5ef99f632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733159738186 2024-12-02T17:15:40,289 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 609eea962027457881921c4c95c5fdb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159739394 2024-12-02T17:15:40,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/9777756af96c47a5a8576638a7bae237 is 50, key is test_row_0/B:col10/1733159740182/Put/seqid=0 2024-12-02T17:15:40,315 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:40,316 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d5f97e046d2a413a99cf9f6d7827447c is 50, key is test_row_0/C:col10/1733159739395/Put/seqid=0 2024-12-02T17:15:40,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742151_1327 (size=12629) 2024-12-02T17:15:40,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159800338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,351 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d5f97e046d2a413a99cf9f6d7827447c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d5f97e046d2a413a99cf9f6d7827447c 2024-12-02T17:15:40,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742150_1326 (size=12301) 2024-12-02T17:15:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-02T17:15:40,353 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159800348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-02T17:15:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T17:15:40,356 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:40,356 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into d5f97e046d2a413a99cf9f6d7827447c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:40,356 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:40,357 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159740180; duration=0sec 2024-12-02T17:15:40,357 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:40,357 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:40,359 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:40,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T17:15:40,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T17:15:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159800557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159800566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T17:15:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T17:15:40,684 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/e7a6442c3f7a42d5abbf687908216dfc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e7a6442c3f7a42d5abbf687908216dfc 2024-12-02T17:15:40,701 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into e7a6442c3f7a42d5abbf687908216dfc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:40,701 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:40,701 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159740180; duration=0sec 2024-12-02T17:15:40,701 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:40,701 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:40,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/9777756af96c47a5a8576638a7bae237 2024-12-02T17:15:40,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/17bb62b5a3544a72b04ad13215f143be is 50, key is test_row_0/C:col10/1733159740182/Put/seqid=0 2024-12-02T17:15:40,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742152_1328 (size=12301) 2024-12-02T17:15:40,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T17:15:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159800867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159800877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T17:15:40,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159800990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:40,993 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8311 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:40,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T17:15:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:40,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:40,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:40,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:41,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T17:15:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:41,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:41,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:41,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:41,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:41,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/17bb62b5a3544a72b04ad13215f143be 2024-12-02T17:15:41,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f21fabb90c9041b3979383fdbea0fcfa as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f21fabb90c9041b3979383fdbea0fcfa 2024-12-02T17:15:41,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f21fabb90c9041b3979383fdbea0fcfa, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T17:15:41,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/9777756af96c47a5a8576638a7bae237 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9777756af96c47a5a8576638a7bae237 2024-12-02T17:15:41,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9777756af96c47a5a8576638a7bae237, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T17:15:41,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/17bb62b5a3544a72b04ad13215f143be as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/17bb62b5a3544a72b04ad13215f143be 2024-12-02T17:15:41,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/17bb62b5a3544a72b04ad13215f143be, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T17:15:41,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for cfac2600ff5eef41b88bb4f556e87599 in 1067ms, sequenceid=281, compaction requested=false 2024-12-02T17:15:41,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:41,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-02T17:15:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:41,308 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:15:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:41,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:41,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/96f4a75fe5774d2ba7452fd0f91e445f is 50, key is test_row_1/A:col10/1733159740210/Put/seqid=0 2024-12-02T17:15:41,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742153_1329 (size=9857) 2024-12-02T17:15:41,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:41,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159741427, max timestamp=1733159741456Num uniques 859; 2024-12-02T17:15:41,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:41,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159741427, max timestamp=1733159741456Num uniques 859; 2024-12-02T17:15:41,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159741427, max timestamp=1733159741456Num uniques 859; 2024-12-02T17:15:41,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:41,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T17:15:41,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159801517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:41,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159801529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:41,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159801637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:41,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159801647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:41,774 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/96f4a75fe5774d2ba7452fd0f91e445f 2024-12-02T17:15:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/4c75362e3dca42c19c582cbc41951f34 is 50, key is test_row_1/B:col10/1733159740210/Put/seqid=0 2024-12-02T17:15:41,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742154_1330 (size=9857) 2024-12-02T17:15:41,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159801855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:41,868 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/4c75362e3dca42c19c582cbc41951f34 2024-12-02T17:15:41,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159801861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:41,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/566d7f98dfcf492aaf2f429af2a54d08 is 50, key is test_row_1/C:col10/1733159740210/Put/seqid=0 2024-12-02T17:15:41,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742155_1331 (size=9857) 2024-12-02T17:15:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159802177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159802180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,357 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/566d7f98dfcf492aaf2f429af2a54d08 2024-12-02T17:15:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/96f4a75fe5774d2ba7452fd0f91e445f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/96f4a75fe5774d2ba7452fd0f91e445f 2024-12-02T17:15:42,379 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/96f4a75fe5774d2ba7452fd0f91e445f, entries=100, sequenceid=294, filesize=9.6 K 2024-12-02T17:15:42,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/4c75362e3dca42c19c582cbc41951f34 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4c75362e3dca42c19c582cbc41951f34 2024-12-02T17:15:42,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4c75362e3dca42c19c582cbc41951f34, entries=100, sequenceid=294, filesize=9.6 K 2024-12-02T17:15:42,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/566d7f98dfcf492aaf2f429af2a54d08 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/566d7f98dfcf492aaf2f429af2a54d08 2024-12-02T17:15:42,389 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/566d7f98dfcf492aaf2f429af2a54d08, entries=100, sequenceid=294, filesize=9.6 K 2024-12-02T17:15:42,390 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cfac2600ff5eef41b88bb4f556e87599 in 1082ms, sequenceid=294, compaction requested=true 2024-12-02T17:15:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-02T17:15:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-02T17:15:42,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-02T17:15:42,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0330 sec 2024-12-02T17:15:42,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0420 sec 2024-12-02T17:15:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-02T17:15:42,507 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:42,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-02T17:15:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:42,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-02T17:15:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:42,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:42,512 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T17:15:42,513 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:42,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:42,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/aa4ce143a4d447b4a910ed8416564e77 is 50, key is test_row_0/A:col10/1733159741521/Put/seqid=0 2024-12-02T17:15:42,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159802538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742156_1332 (size=14741) 2024-12-02T17:15:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159802540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/aa4ce143a4d447b4a910ed8416564e77 2024-12-02T17:15:42,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/24ea937cc3e14f23bac157f149faef17 is 50, key is test_row_0/B:col10/1733159741521/Put/seqid=0 2024-12-02T17:15:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T17:15:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742157_1333 (size=12301) 2024-12-02T17:15:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159802647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159802657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T17:15:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159802687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159802697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T17:15:42,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:42,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T17:15:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159802857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159802866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:42,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:42,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T17:15:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:42,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:42,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:43,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/24ea937cc3e14f23bac157f149faef17 2024-12-02T17:15:43,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/9c0ecfc161544be995c8b06404da0ec8 is 50, key is test_row_0/C:col10/1733159741521/Put/seqid=0 2024-12-02T17:15:43,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742158_1334 (size=12301) 2024-12-02T17:15:43,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/9c0ecfc161544be995c8b06404da0ec8 2024-12-02T17:15:43,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/aa4ce143a4d447b4a910ed8416564e77 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/aa4ce143a4d447b4a910ed8416564e77 2024-12-02T17:15:43,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/aa4ce143a4d447b4a910ed8416564e77, entries=200, sequenceid=331, filesize=14.4 K 2024-12-02T17:15:43,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/24ea937cc3e14f23bac157f149faef17 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/24ea937cc3e14f23bac157f149faef17 2024-12-02T17:15:43,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/24ea937cc3e14f23bac157f149faef17, entries=150, sequenceid=331, filesize=12.0 K 2024-12-02T17:15:43,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/9c0ecfc161544be995c8b06404da0ec8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/9c0ecfc161544be995c8b06404da0ec8 2024-12-02T17:15:43,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/9c0ecfc161544be995c8b06404da0ec8, entries=150, sequenceid=331, filesize=12.0 K 2024-12-02T17:15:43,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cfac2600ff5eef41b88bb4f556e87599 in 569ms, sequenceid=331, compaction requested=true 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:43,081 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:43,081 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:43,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:43,083 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:43,083 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:43,083 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,083 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e7a6442c3f7a42d5abbf687908216dfc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9777756af96c47a5a8576638a7bae237, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4c75362e3dca42c19c582cbc41951f34, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/24ea937cc3e14f23bac157f149faef17] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=46.0 K 2024-12-02T17:15:43,083 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49528 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:43,084 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:43,084 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,084 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8dab7d824fcf4631b7b7ab6ebc201213, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f21fabb90c9041b3979383fdbea0fcfa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/96f4a75fe5774d2ba7452fd0f91e445f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/aa4ce143a4d447b4a910ed8416564e77] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=48.4 K 2024-12-02T17:15:43,084 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a6442c3f7a42d5abbf687908216dfc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159739394 2024-12-02T17:15:43,084 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dab7d824fcf4631b7b7ab6ebc201213, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159739394 2024-12-02T17:15:43,085 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9777756af96c47a5a8576638a7bae237, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733159740166 2024-12-02T17:15:43,085 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f21fabb90c9041b3979383fdbea0fcfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733159740166 2024-12-02T17:15:43,085 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f4a75fe5774d2ba7452fd0f91e445f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733159740210 2024-12-02T17:15:43,085 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c75362e3dca42c19c582cbc41951f34, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733159740210 2024-12-02T17:15:43,086 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4ce143a4d447b4a910ed8416564e77, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733159741497 2024-12-02T17:15:43,086 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ea937cc3e14f23bac157f149faef17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733159741497 2024-12-02T17:15:43,099 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:43,100 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/2d4098fd0f6049eca984adfe141f5805 is 50, key is test_row_0/B:col10/1733159741521/Put/seqid=0 2024-12-02T17:15:43,103 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:43,103 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/dae71028e5fc425eba5106399e09b5ae is 50, key is test_row_0/A:col10/1733159741521/Put/seqid=0 2024-12-02T17:15:43,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742160_1336 (size=12915) 2024-12-02T17:15:43,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742159_1335 (size=12915) 2024-12-02T17:15:43,126 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/dae71028e5fc425eba5106399e09b5ae as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/dae71028e5fc425eba5106399e09b5ae 2024-12-02T17:15:43,126 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/2d4098fd0f6049eca984adfe141f5805 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/2d4098fd0f6049eca984adfe141f5805 2024-12-02T17:15:43,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-02T17:15:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,127 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T17:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:43,135 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into dae71028e5fc425eba5106399e09b5ae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:43,135 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,135 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=12, startTime=1733159743081; duration=0sec 2024-12-02T17:15:43,135 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:43,135 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:43,135 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:43,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T17:15:43,137 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 2d4098fd0f6049eca984adfe141f5805(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:43,137 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,137 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=12, startTime=1733159743081; duration=0sec 2024-12-02T17:15:43,137 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:43,137 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:43,137 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:43,137 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:43,137 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,137 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d5f97e046d2a413a99cf9f6d7827447c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/17bb62b5a3544a72b04ad13215f143be, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/566d7f98dfcf492aaf2f429af2a54d08, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/9c0ecfc161544be995c8b06404da0ec8] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=46.0 K 2024-12-02T17:15:43,138 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5f97e046d2a413a99cf9f6d7827447c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733159739394 2024-12-02T17:15:43,138 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17bb62b5a3544a72b04ad13215f143be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733159740166 2024-12-02T17:15:43,138 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566d7f98dfcf492aaf2f429af2a54d08, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733159740210 2024-12-02T17:15:43,139 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c0ecfc161544be995c8b06404da0ec8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733159741497 2024-12-02T17:15:43,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f86f63af5787472c90b4fc4cf74e15b8 is 50, key is test_row_0/A:col10/1733159742523/Put/seqid=0 2024-12-02T17:15:43,163 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:43,164 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/64bae6e2e47645db9f4c82d8ea11b730 is 50, key is test_row_0/C:col10/1733159741521/Put/seqid=0 2024-12-02T17:15:43,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742162_1338 (size=12915) 2024-12-02T17:15:43,202 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/64bae6e2e47645db9f4c82d8ea11b730 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/64bae6e2e47645db9f4c82d8ea11b730 2024-12-02T17:15:43,209 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 64bae6e2e47645db9f4c82d8ea11b730(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:43,210 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,210 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=12, startTime=1733159743081; duration=0sec 2024-12-02T17:15:43,210 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:43,210 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:43,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742161_1337 (size=12301) 2024-12-02T17:15:43,219 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f86f63af5787472c90b4fc4cf74e15b8 2024-12-02T17:15:43,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/87ec764f096942889e39394ff9525005 is 50, key is test_row_0/B:col10/1733159742523/Put/seqid=0 2024-12-02T17:15:43,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159743235, max timestamp=1733159743251Num uniques 859; 2024-12-02T17:15:43,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159743235, max timestamp=1733159743251Num uniques 859; 2024-12-02T17:15:43,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:43,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159743235, max timestamp=1733159743251Num uniques 859; 2024-12-02T17:15:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742163_1339 (size=12301) 2024-12-02T17:15:43,259 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/87ec764f096942889e39394ff9525005 2024-12-02T17:15:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/afe912b1608f48f1b0b3a0fe2b0e7a19 is 50, key is test_row_0/C:col10/1733159742523/Put/seqid=0 2024-12-02T17:15:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742164_1340 (size=12301) 2024-12-02T17:15:43,298 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/afe912b1608f48f1b0b3a0fe2b0e7a19 2024-12-02T17:15:43,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f86f63af5787472c90b4fc4cf74e15b8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f86f63af5787472c90b4fc4cf74e15b8 2024-12-02T17:15:43,311 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f86f63af5787472c90b4fc4cf74e15b8, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T17:15:43,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/87ec764f096942889e39394ff9525005 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/87ec764f096942889e39394ff9525005 2024-12-02T17:15:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/87ec764f096942889e39394ff9525005, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T17:15:43,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/afe912b1608f48f1b0b3a0fe2b0e7a19 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/afe912b1608f48f1b0b3a0fe2b0e7a19 2024-12-02T17:15:43,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/afe912b1608f48f1b0b3a0fe2b0e7a19, entries=150, sequenceid=340, filesize=12.0 K 2024-12-02T17:15:43,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=120.76 KB/123660 for cfac2600ff5eef41b88bb4f556e87599 in 197ms, sequenceid=340, compaction requested=false 2024-12-02T17:15:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-02T17:15:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-02T17:15:43,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-02T17:15:43,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 812 msec 2024-12-02T17:15:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:43,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-02T17:15:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:43,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 821 msec 2024-12-02T17:15:43,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f621af18d7984cb4bcbdee517b4e3b01 is 50, key is test_row_0/A:col10/1733159743302/Put/seqid=0 2024-12-02T17:15:43,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742165_1341 (size=12301) 2024-12-02T17:15:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159803396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159803398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159803507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159803507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-02T17:15:43,647 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:43,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-02T17:15:43,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T17:15:43,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:43,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:43,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:43,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159803697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,703 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159803717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159803717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,721 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:43,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159803727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T17:15:43,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f621af18d7984cb4bcbdee517b4e3b01 2024-12-02T17:15:43,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/4d90cf336a3645ba9411b14e43a2076b is 50, key is test_row_0/B:col10/1733159743302/Put/seqid=0 2024-12-02T17:15:43,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742166_1342 (size=12301) 2024-12-02T17:15:43,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T17:15:43,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/4d90cf336a3645ba9411b14e43a2076b 2024-12-02T17:15:43,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:43,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:43,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:43,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/f58f2e753e9148b1a43363ac4499571f is 50, key is test_row_0/C:col10/1733159743302/Put/seqid=0 2024-12-02T17:15:43,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742167_1343 (size=12301) 2024-12-02T17:15:43,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/f58f2e753e9148b1a43363ac4499571f 2024-12-02T17:15:43,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/f621af18d7984cb4bcbdee517b4e3b01 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f621af18d7984cb4bcbdee517b4e3b01 2024-12-02T17:15:43,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f621af18d7984cb4bcbdee517b4e3b01, entries=150, sequenceid=379, filesize=12.0 K 2024-12-02T17:15:43,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/4d90cf336a3645ba9411b14e43a2076b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4d90cf336a3645ba9411b14e43a2076b 2024-12-02T17:15:43,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4d90cf336a3645ba9411b14e43a2076b, entries=150, sequenceid=379, filesize=12.0 K 2024-12-02T17:15:43,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/f58f2e753e9148b1a43363ac4499571f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/f58f2e753e9148b1a43363ac4499571f 2024-12-02T17:15:43,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/f58f2e753e9148b1a43363ac4499571f, entries=150, sequenceid=379, filesize=12.0 K 2024-12-02T17:15:43,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cfac2600ff5eef41b88bb4f556e87599 in 546ms, sequenceid=379, compaction requested=true 2024-12-02T17:15:43,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,875 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:43,877 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:43,877 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:43,877 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,877 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/dae71028e5fc425eba5106399e09b5ae, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f86f63af5787472c90b4fc4cf74e15b8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f621af18d7984cb4bcbdee517b4e3b01] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.6 K 2024-12-02T17:15:43,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:43,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:43,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:43,878 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:43,878 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae71028e5fc425eba5106399e09b5ae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733159741497 2024-12-02T17:15:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:43,879 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86f63af5787472c90b4fc4cf74e15b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159742523 2024-12-02T17:15:43,880 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:43,880 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:43,880 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,880 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/2d4098fd0f6049eca984adfe141f5805, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/87ec764f096942889e39394ff9525005, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4d90cf336a3645ba9411b14e43a2076b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.6 K 2024-12-02T17:15:43,880 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f621af18d7984cb4bcbdee517b4e3b01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733159743302 2024-12-02T17:15:43,881 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4098fd0f6049eca984adfe141f5805, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733159741497 2024-12-02T17:15:43,882 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ec764f096942889e39394ff9525005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159742523 2024-12-02T17:15:43,884 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d90cf336a3645ba9411b14e43a2076b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733159743302 2024-12-02T17:15:43,897 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:43,898 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/8489dfd9c7a344b59eda8c911c6d4d69 is 50, key is test_row_0/A:col10/1733159743302/Put/seqid=0 2024-12-02T17:15:43,910 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:43,911 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/276ddb7534a4417d97476249eb0dba9c is 50, key is test_row_0/B:col10/1733159743302/Put/seqid=0 2024-12-02T17:15:43,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742168_1344 (size=13017) 2024-12-02T17:15:43,953 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/8489dfd9c7a344b59eda8c911c6d4d69 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8489dfd9c7a344b59eda8c911c6d4d69 2024-12-02T17:15:43,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-02T17:15:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,958 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:15:43,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:43,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:43,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:43,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:43,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:43,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:43,963 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 8489dfd9c7a344b59eda8c911c6d4d69(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:43,963 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:43,963 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159743875; duration=0sec 2024-12-02T17:15:43,963 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:43,963 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:43,963 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:43,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742169_1345 (size=13017) 2024-12-02T17:15:43,965 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:43,968 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:43,968 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:43,968 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/64bae6e2e47645db9f4c82d8ea11b730, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/afe912b1608f48f1b0b3a0fe2b0e7a19, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/f58f2e753e9148b1a43363ac4499571f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.6 K 2024-12-02T17:15:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T17:15:43,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/d47e7211a0704e598a94b2dc5a76aead is 50, key is test_row_0/A:col10/1733159743351/Put/seqid=0 2024-12-02T17:15:43,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64bae6e2e47645db9f4c82d8ea11b730, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733159741497 2024-12-02T17:15:43,970 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe912b1608f48f1b0b3a0fe2b0e7a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733159742523 2024-12-02T17:15:43,971 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58f2e753e9148b1a43363ac4499571f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733159743302 2024-12-02T17:15:44,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742170_1346 (size=12301) 2024-12-02T17:15:44,008 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/d47e7211a0704e598a94b2dc5a76aead 2024-12-02T17:15:44,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:44,040 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:44,041 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/bcb12938b62e412e9b4a72a8926aaaff is 50, key is test_row_0/C:col10/1733159743302/Put/seqid=0 2024-12-02T17:15:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/985433b48c434636a6a8b81b77e99d85 is 50, key is test_row_0/B:col10/1733159743351/Put/seqid=0 2024-12-02T17:15:44,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742171_1347 (size=12301) 2024-12-02T17:15:44,092 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/985433b48c434636a6a8b81b77e99d85 2024-12-02T17:15:44,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742172_1348 (size=13017) 2024-12-02T17:15:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b7a049bab09148fe9b95fc5675b48384 is 50, key is test_row_0/C:col10/1733159743351/Put/seqid=0 2024-12-02T17:15:44,116 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/bcb12938b62e412e9b4a72a8926aaaff as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/bcb12938b62e412e9b4a72a8926aaaff 2024-12-02T17:15:44,121 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into bcb12938b62e412e9b4a72a8926aaaff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:44,121 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:44,121 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159743878; duration=0sec 2024-12-02T17:15:44,122 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:44,122 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:44,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742173_1349 (size=12301) 2024-12-02T17:15:44,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159744113, max timestamp=1733159744161Num uniques 859; 2024-12-02T17:15:44,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:44,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159744113, max timestamp=1733159744161Num uniques 859; 2024-12-02T17:15:44,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:15:44,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733159744113, max timestamp=1733159744161Num uniques 859; 2024-12-02T17:15:44,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-02T17:15:44,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159804241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159804246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T17:15:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159804358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159804367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,371 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/276ddb7534a4417d97476249eb0dba9c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/276ddb7534a4417d97476249eb0dba9c 2024-12-02T17:15:44,380 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 276ddb7534a4417d97476249eb0dba9c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:44,380 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:44,380 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159743877; duration=0sec 2024-12-02T17:15:44,380 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:44,380 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:44,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b7a049bab09148fe9b95fc5675b48384 2024-12-02T17:15:44,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/d47e7211a0704e598a94b2dc5a76aead as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d47e7211a0704e598a94b2dc5a76aead 2024-12-02T17:15:44,568 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d47e7211a0704e598a94b2dc5a76aead, entries=150, sequenceid=390, filesize=12.0 K 2024-12-02T17:15:44,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/985433b48c434636a6a8b81b77e99d85 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/985433b48c434636a6a8b81b77e99d85 2024-12-02T17:15:44,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159804567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159804576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,582 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/985433b48c434636a6a8b81b77e99d85, entries=150, sequenceid=390, filesize=12.0 K 2024-12-02T17:15:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b7a049bab09148fe9b95fc5675b48384 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b7a049bab09148fe9b95fc5675b48384 2024-12-02T17:15:44,590 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b7a049bab09148fe9b95fc5675b48384, entries=150, sequenceid=390, filesize=12.0 K 2024-12-02T17:15:44,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for cfac2600ff5eef41b88bb4f556e87599 in 633ms, sequenceid=390, compaction requested=false 2024-12-02T17:15:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:44,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-02T17:15:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-02T17:15:44,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-02T17:15:44,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 942 msec 2024-12-02T17:15:44,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 951 msec 2024-12-02T17:15:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-02T17:15:44,792 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:44,792 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-02T17:15:44,795 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T17:15:44,796 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:44,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:44,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-12-02T17:15:44,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:44,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:15:44,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:44,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:15:44,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:44,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:15:44,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/cef7f7fbda3148bc92446bc14a2959ec is 50, key is test_row_0/A:col10/1733159744232/Put/seqid=0 2024-12-02T17:15:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T17:15:44,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742174_1350 (size=12301) 2024-12-02T17:15:44,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159804919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:44,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159804927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:44,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:44,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:44,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159805037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159805037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:45,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T17:15:45,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159805247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159805248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/cef7f7fbda3148bc92446bc14a2959ec 2024-12-02T17:15:45,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/7779a71f56394db9b8c493b6822bad94 is 50, key is test_row_0/B:col10/1733159744232/Put/seqid=0 2024-12-02T17:15:45,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742175_1351 (size=12301) 2024-12-02T17:15:45,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T17:15:45,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159805557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159805566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159805721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,728 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:45,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:45,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159805748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:45,757 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:45,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:45,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/7779a71f56394db9b8c493b6822bad94 2024-12-02T17:15:45,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/60612c93b06244c6a7a056a8c771fe88 is 50, key is test_row_0/C:col10/1733159744232/Put/seqid=0 2024-12-02T17:15:45,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:45,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742176_1352 (size=12301) 2024-12-02T17:15:45,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/60612c93b06244c6a7a056a8c771fe88 2024-12-02T17:15:45,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/cef7f7fbda3148bc92446bc14a2959ec as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/cef7f7fbda3148bc92446bc14a2959ec 2024-12-02T17:15:45,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/cef7f7fbda3148bc92446bc14a2959ec, entries=150, sequenceid=431, filesize=12.0 K 2024-12-02T17:15:45,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/7779a71f56394db9b8c493b6822bad94 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7779a71f56394db9b8c493b6822bad94 2024-12-02T17:15:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T17:15:45,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7779a71f56394db9b8c493b6822bad94, entries=150, sequenceid=431, filesize=12.0 K 2024-12-02T17:15:45,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/60612c93b06244c6a7a056a8c771fe88 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/60612c93b06244c6a7a056a8c771fe88 2024-12-02T17:15:45,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/60612c93b06244c6a7a056a8c771fe88, entries=150, sequenceid=431, filesize=12.0 K 2024-12-02T17:15:45,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.10 KB/143460, heapSize ~403.73 KB/413424, currentSize=40.25 KB/41220 for cfac2600ff5eef41b88bb4f556e87599 in 1056ms, sequenceid=431, compaction requested=true 2024-12-02T17:15:45,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:45,951 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:45,953 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:45,953 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:45,953 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,953 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8489dfd9c7a344b59eda8c911c6d4d69, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d47e7211a0704e598a94b2dc5a76aead, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/cef7f7fbda3148bc92446bc14a2959ec] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.7 K 2024-12-02T17:15:45,953 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8489dfd9c7a344b59eda8c911c6d4d69, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733159743302 2024-12-02T17:15:45,954 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47e7211a0704e598a94b2dc5a76aead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733159743351 2024-12-02T17:15:45,954 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef7f7fbda3148bc92446bc14a2959ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733159744232 2024-12-02T17:15:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:45,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:15:45,975 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:45,978 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:45,978 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:45,978 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:45,978 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:45,978 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/276ddb7534a4417d97476249eb0dba9c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/985433b48c434636a6a8b81b77e99d85, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7779a71f56394db9b8c493b6822bad94] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.7 K 2024-12-02T17:15:45,978 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/c43e8bb76746466892ac3276f8ce871b is 50, key is test_row_0/A:col10/1733159744232/Put/seqid=0 2024-12-02T17:15:45,979 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 276ddb7534a4417d97476249eb0dba9c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733159743302 2024-12-02T17:15:45,979 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 985433b48c434636a6a8b81b77e99d85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733159743351 2024-12-02T17:15:45,980 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 7779a71f56394db9b8c493b6822bad94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733159744232 2024-12-02T17:15:45,992 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:45,992 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/be1d72c7fb904c67a42e855695c291fa is 50, key is test_row_0/B:col10/1733159744232/Put/seqid=0 2024-12-02T17:15:46,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742177_1353 (size=13119) 2024-12-02T17:15:46,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742178_1354 (size=13119) 2024-12-02T17:15:46,033 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/c43e8bb76746466892ac3276f8ce871b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/c43e8bb76746466892ac3276f8ce871b 2024-12-02T17:15:46,038 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into c43e8bb76746466892ac3276f8ce871b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:46,038 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:46,038 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159745951; duration=0sec 2024-12-02T17:15:46,038 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:46,039 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:46,039 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:46,040 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:46,040 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:46,040 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:46,040 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/bcb12938b62e412e9b4a72a8926aaaff, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b7a049bab09148fe9b95fc5675b48384, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/60612c93b06244c6a7a056a8c771fe88] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.7 K 2024-12-02T17:15:46,040 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb12938b62e412e9b4a72a8926aaaff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733159743302 2024-12-02T17:15:46,041 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a049bab09148fe9b95fc5675b48384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1733159743351 2024-12-02T17:15:46,041 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60612c93b06244c6a7a056a8c771fe88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733159744232 2024-12-02T17:15:46,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-02T17:15:46,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:46,059 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-02T17:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:46,063 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:46,064 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/3a8897fdf4ca4593b5441d3b886f41e8 is 50, key is test_row_0/C:col10/1733159744232/Put/seqid=0 2024-12-02T17:15:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/65dd4d430a0947b4bb6d2a06788d1cf4 is 50, key is test_row_0/A:col10/1733159744901/Put/seqid=0 2024-12-02T17:15:46,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:46,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742179_1355 (size=13119) 2024-12-02T17:15:46,112 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/3a8897fdf4ca4593b5441d3b886f41e8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/3a8897fdf4ca4593b5441d3b886f41e8 2024-12-02T17:15:46,123 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 3a8897fdf4ca4593b5441d3b886f41e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:46,123 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:46,123 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159745973; duration=0sec 2024-12-02T17:15:46,123 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:46,123 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:46,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742180_1356 (size=12301) 2024-12-02T17:15:46,142 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/65dd4d430a0947b4bb6d2a06788d1cf4 2024-12-02T17:15:46,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/67fd6413a79c4db69503c56d294605a0 is 50, key is test_row_0/B:col10/1733159744901/Put/seqid=0 2024-12-02T17:15:46,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742181_1357 (size=12301) 2024-12-02T17:15:46,209 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/67fd6413a79c4db69503c56d294605a0 2024-12-02T17:15:46,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159746151, max timestamp=1733159746219Num uniques 859; 2024-12-02T17:15:46,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:46,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159746151, max timestamp=1733159746219Num uniques 859; 2024-12-02T17:15:46,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:46,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159746151, max timestamp=1733159746219Num uniques 859; 2024-12-02T17:15:46,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:15:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/077a525ab2134e2098ac29f8cdbb5630 is 50, key is test_row_0/C:col10/1733159744901/Put/seqid=0 2024-12-02T17:15:46,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742182_1358 (size=12301) 2024-12-02T17:15:46,280 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/077a525ab2134e2098ac29f8cdbb5630 2024-12-02T17:15:46,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/65dd4d430a0947b4bb6d2a06788d1cf4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/65dd4d430a0947b4bb6d2a06788d1cf4 2024-12-02T17:15:46,295 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/65dd4d430a0947b4bb6d2a06788d1cf4, entries=150, sequenceid=441, filesize=12.0 K 2024-12-02T17:15:46,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/67fd6413a79c4db69503c56d294605a0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/67fd6413a79c4db69503c56d294605a0 2024-12-02T17:15:46,307 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/67fd6413a79c4db69503c56d294605a0, entries=150, sequenceid=441, filesize=12.0 K 2024-12-02T17:15:46,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/077a525ab2134e2098ac29f8cdbb5630 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/077a525ab2134e2098ac29f8cdbb5630 2024-12-02T17:15:46,313 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/077a525ab2134e2098ac29f8cdbb5630, entries=150, sequenceid=441, filesize=12.0 K 2024-12-02T17:15:46,325 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=100.63 KB/103050 for cfac2600ff5eef41b88bb4f556e87599 in 266ms, sequenceid=441, compaction requested=false 2024-12-02T17:15:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:46,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-02T17:15:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-02T17:15:46,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-02T17:15:46,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5310 sec 2024-12-02T17:15:46,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5370 sec 2024-12-02T17:15:46,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=107.34 KB heapSize=305.30 KB 2024-12-02T17:15:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:46,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:46,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:46,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:46,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:46,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:46,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:46,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/81a87451899d4581a893a1331b3dcc35 is 50, key is test_row_0/A:col10/1733159746337/Put/seqid=0 2024-12-02T17:15:46,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742183_1359 (size=14741) 2024-12-02T17:15:46,477 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/be1d72c7fb904c67a42e855695c291fa as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/be1d72c7fb904c67a42e855695c291fa 2024-12-02T17:15:46,483 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into be1d72c7fb904c67a42e855695c291fa(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:46,483 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:46,483 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159745973; duration=0sec 2024-12-02T17:15:46,484 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:46,484 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:46,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159806496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159806498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159806629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:46,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159806629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:46,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/81a87451899d4581a893a1331b3dcc35 2024-12-02T17:15:46,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/6970d6ad097b4b7b97b7bb90eb56f906 is 50, key is test_row_0/B:col10/1733159746337/Put/seqid=0 2024-12-02T17:15:46,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159806849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159806861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:46,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742184_1360 (size=12301) 2024-12-02T17:15:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-02T17:15:46,949 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:46,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:46,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-02T17:15:46,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:46,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T17:15:46,954 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:46,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T17:15:47,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T17:15:47,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:47,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159807178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:47,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159807187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:47,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T17:15:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T17:15:47,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/6970d6ad097b4b7b97b7bb90eb56f906 2024-12-02T17:15:47,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/c5f9675cfd714c73a4cb241b99e454fe is 50, key is test_row_0/C:col10/1733159746337/Put/seqid=0 2024-12-02T17:15:47,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742185_1361 (size=12301) 2024-12-02T17:15:47,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/c5f9675cfd714c73a4cb241b99e454fe 2024-12-02T17:15:47,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T17:15:47,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:47,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:47,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/81a87451899d4581a893a1331b3dcc35 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/81a87451899d4581a893a1331b3dcc35 2024-12-02T17:15:47,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/81a87451899d4581a893a1331b3dcc35, entries=200, sequenceid=476, filesize=14.4 K 2024-12-02T17:15:47,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/6970d6ad097b4b7b97b7bb90eb56f906 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6970d6ad097b4b7b97b7bb90eb56f906 2024-12-02T17:15:47,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6970d6ad097b4b7b97b7bb90eb56f906, entries=150, sequenceid=476, filesize=12.0 K 2024-12-02T17:15:47,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/c5f9675cfd714c73a4cb241b99e454fe as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/c5f9675cfd714c73a4cb241b99e454fe 2024-12-02T17:15:47,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/c5f9675cfd714c73a4cb241b99e454fe, entries=150, sequenceid=476, filesize=12.0 K 2024-12-02T17:15:47,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for cfac2600ff5eef41b88bb4f556e87599 in 1147ms, sequenceid=476, compaction requested=true 2024-12-02T17:15:47,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:47,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:47,487 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:47,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:47,489 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:47,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:47,489 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:47,489 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:47,489 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,490 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/c43e8bb76746466892ac3276f8ce871b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/65dd4d430a0947b4bb6d2a06788d1cf4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/81a87451899d4581a893a1331b3dcc35] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=39.2 K 2024-12-02T17:15:47,490 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c43e8bb76746466892ac3276f8ce871b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733159744232 2024-12-02T17:15:47,490 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65dd4d430a0947b4bb6d2a06788d1cf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733159744901 2024-12-02T17:15:47,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:47,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:47,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:47,497 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:47,497 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:47,497 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,497 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/be1d72c7fb904c67a42e855695c291fa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/67fd6413a79c4db69503c56d294605a0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6970d6ad097b4b7b97b7bb90eb56f906] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.8 K 2024-12-02T17:15:47,497 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81a87451899d4581a893a1331b3dcc35, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733159746272 2024-12-02T17:15:47,512 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting be1d72c7fb904c67a42e855695c291fa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733159744232 2024-12-02T17:15:47,513 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fd6413a79c4db69503c56d294605a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733159744901 2024-12-02T17:15:47,514 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6970d6ad097b4b7b97b7bb90eb56f906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733159746272 2024-12-02T17:15:47,536 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:47,536 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/71ca618d19d6487d9eca9776a07e7e3a is 50, key is test_row_0/B:col10/1733159746337/Put/seqid=0 2024-12-02T17:15:47,547 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:47,547 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/ebbbf8be8f6c47f68966360cf9cbac1c is 50, key is test_row_0/A:col10/1733159746337/Put/seqid=0 2024-12-02T17:15:47,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,575 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:47,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742186_1362 (size=13221) 2024-12-02T17:15:47,590 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/71ca618d19d6487d9eca9776a07e7e3a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/71ca618d19d6487d9eca9776a07e7e3a 2024-12-02T17:15:47,596 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 71ca618d19d6487d9eca9776a07e7e3a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:47,596 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T17:15:47,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/da7fcca850414d9d82fbd4cd61f7fefe is 50, key is test_row_0/A:col10/1733159746496/Put/seqid=0 2024-12-02T17:15:47,598 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159747489; duration=0sec 2024-12-02T17:15:47,598 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:47,598 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:47,598 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742187_1363 (size=13221) 2024-12-02T17:15:47,600 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:47,600 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:47,600 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:47,600 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/3a8897fdf4ca4593b5441d3b886f41e8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/077a525ab2134e2098ac29f8cdbb5630, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/c5f9675cfd714c73a4cb241b99e454fe] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.8 K 2024-12-02T17:15:47,601 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8897fdf4ca4593b5441d3b886f41e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1733159744232 2024-12-02T17:15:47,601 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 077a525ab2134e2098ac29f8cdbb5630, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1733159744901 2024-12-02T17:15:47,602 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f9675cfd714c73a4cb241b99e454fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733159746272 2024-12-02T17:15:47,606 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/ebbbf8be8f6c47f68966360cf9cbac1c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/ebbbf8be8f6c47f68966360cf9cbac1c 2024-12-02T17:15:47,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742188_1364 (size=12301) 2024-12-02T17:15:47,618 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into ebbbf8be8f6c47f68966360cf9cbac1c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:47,618 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:47,618 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159747487; duration=0sec 2024-12-02T17:15:47,619 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:47,619 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:47,620 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/da7fcca850414d9d82fbd4cd61f7fefe 2024-12-02T17:15:47,635 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:47,635 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/4ca3c38e403740dc932d5c928ae53c25 is 50, key is test_row_0/C:col10/1733159746337/Put/seqid=0 2024-12-02T17:15:47,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/e2ac5b93df0c49b9a04679a94d7839e0 is 50, key is test_row_0/B:col10/1733159746496/Put/seqid=0 2024-12-02T17:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:47,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:47,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742190_1366 (size=12301) 2024-12-02T17:15:47,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742189_1365 (size=13221) 2024-12-02T17:15:47,718 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/4ca3c38e403740dc932d5c928ae53c25 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/4ca3c38e403740dc932d5c928ae53c25 2024-12-02T17:15:47,756 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 4ca3c38e403740dc932d5c928ae53c25(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:47,756 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:47,756 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159747491; duration=0sec 2024-12-02T17:15:47,756 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:47,756 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:47,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159807812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:47,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159807815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:47,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159807929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159807929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/e2ac5b93df0c49b9a04679a94d7839e0 2024-12-02T17:15:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T17:15:48,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/71319ee568fc48969abd2e6021b7bb55 is 50, key is test_row_0/C:col10/1733159746496/Put/seqid=0 2024-12-02T17:15:48,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159808141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159808147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742191_1367 (size=12301) 2024-12-02T17:15:48,195 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/71319ee568fc48969abd2e6021b7bb55 2024-12-02T17:15:48,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/da7fcca850414d9d82fbd4cd61f7fefe as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/da7fcca850414d9d82fbd4cd61f7fefe 2024-12-02T17:15:48,205 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/da7fcca850414d9d82fbd4cd61f7fefe, entries=150, sequenceid=490, filesize=12.0 K 2024-12-02T17:15:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/e2ac5b93df0c49b9a04679a94d7839e0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e2ac5b93df0c49b9a04679a94d7839e0 2024-12-02T17:15:48,212 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e2ac5b93df0c49b9a04679a94d7839e0, entries=150, sequenceid=490, filesize=12.0 K 2024-12-02T17:15:48,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/71319ee568fc48969abd2e6021b7bb55 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/71319ee568fc48969abd2e6021b7bb55 2024-12-02T17:15:48,219 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/71319ee568fc48969abd2e6021b7bb55, entries=150, sequenceid=490, filesize=12.0 K 2024-12-02T17:15:48,220 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cfac2600ff5eef41b88bb4f556e87599 in 645ms, sequenceid=490, compaction requested=false 2024-12-02T17:15:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:48,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-02T17:15:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-02T17:15:48,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-02T17:15:48,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2680 sec 2024-12-02T17:15:48,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.2790 sec 2024-12-02T17:15:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:48,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:15:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:48,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:48,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2a6230c76dfc4ebe800d35f1cce610f5 is 50, key is test_row_0/A:col10/1733159748458/Put/seqid=0 2024-12-02T17:15:48,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159808512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159808522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742192_1368 (size=12301) 2024-12-02T17:15:48,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159808626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159808637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159808837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159808847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:48,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2a6230c76dfc4ebe800d35f1cce610f5 2024-12-02T17:15:48,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/373f3d775e224f9fab99ae359c009714 is 50, key is test_row_0/B:col10/1733159748458/Put/seqid=0 2024-12-02T17:15:49,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742193_1369 (size=12301) 2024-12-02T17:15:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-02T17:15:49,127 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:49,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-02T17:15:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T17:15:49,133 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:49,133 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:49,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159809147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159809169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T17:15:49,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T17:15:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:49,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/373f3d775e224f9fab99ae359c009714 2024-12-02T17:15:49,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/52d3557cbe014d2e8cc9a49ae8d70899 is 50, key is test_row_0/C:col10/1733159748458/Put/seqid=0 2024-12-02T17:15:49,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T17:15:49,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:49,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:49,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T17:15:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742194_1370 (size=12301) 2024-12-02T17:15:49,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/52d3557cbe014d2e8cc9a49ae8d70899 2024-12-02T17:15:49,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2a6230c76dfc4ebe800d35f1cce610f5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2a6230c76dfc4ebe800d35f1cce610f5 2024-12-02T17:15:49,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2a6230c76dfc4ebe800d35f1cce610f5, entries=150, sequenceid=513, filesize=12.0 K 2024-12-02T17:15:49,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/373f3d775e224f9fab99ae359c009714 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/373f3d775e224f9fab99ae359c009714 2024-12-02T17:15:49,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/373f3d775e224f9fab99ae359c009714, entries=150, sequenceid=513, filesize=12.0 K 2024-12-02T17:15:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/52d3557cbe014d2e8cc9a49ae8d70899 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/52d3557cbe014d2e8cc9a49ae8d70899 2024-12-02T17:15:49,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/52d3557cbe014d2e8cc9a49ae8d70899, entries=150, sequenceid=513, filesize=12.0 K 2024-12-02T17:15:49,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cfac2600ff5eef41b88bb4f556e87599 in 1053ms, sequenceid=513, compaction requested=true 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:49,516 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:49,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:49,516 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:49,517 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:49,517 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:49,517 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,517 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/ebbbf8be8f6c47f68966360cf9cbac1c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/da7fcca850414d9d82fbd4cd61f7fefe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2a6230c76dfc4ebe800d35f1cce610f5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.9 K 2024-12-02T17:15:49,518 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:49,519 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:49,519 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,519 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/71ca618d19d6487d9eca9776a07e7e3a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e2ac5b93df0c49b9a04679a94d7839e0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/373f3d775e224f9fab99ae359c009714] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.9 K 2024-12-02T17:15:49,519 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebbbf8be8f6c47f68966360cf9cbac1c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733159746272 2024-12-02T17:15:49,519 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ca618d19d6487d9eca9776a07e7e3a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733159746272 2024-12-02T17:15:49,520 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ac5b93df0c49b9a04679a94d7839e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733159746432 2024-12-02T17:15:49,520 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7fcca850414d9d82fbd4cd61f7fefe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733159746432 2024-12-02T17:15:49,520 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 373f3d775e224f9fab99ae359c009714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733159747793 2024-12-02T17:15:49,522 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6230c76dfc4ebe800d35f1cce610f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733159747793 2024-12-02T17:15:49,533 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:49,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/75c0e0001df6496f9f74240bbe838dee is 50, key is test_row_0/B:col10/1733159748458/Put/seqid=0 2024-12-02T17:15:49,539 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:49,543 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2f2678bbb225468b95b597de023cc4a2 is 50, key is test_row_0/A:col10/1733159748458/Put/seqid=0 2024-12-02T17:15:49,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742195_1371 (size=13323) 2024-12-02T17:15:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742196_1372 (size=13323) 2024-12-02T17:15:49,577 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/2f2678bbb225468b95b597de023cc4a2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2f2678bbb225468b95b597de023cc4a2 2024-12-02T17:15:49,584 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 2f2678bbb225468b95b597de023cc4a2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:49,584 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:49,584 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159749516; duration=0sec 2024-12-02T17:15:49,585 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:49,585 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:49,585 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:49,585 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:49,586 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:49,586 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,586 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/4ca3c38e403740dc932d5c928ae53c25, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/71319ee568fc48969abd2e6021b7bb55, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/52d3557cbe014d2e8cc9a49ae8d70899] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=36.9 K 2024-12-02T17:15:49,586 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ca3c38e403740dc932d5c928ae53c25, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733159746272 2024-12-02T17:15:49,586 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71319ee568fc48969abd2e6021b7bb55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1733159746432 2024-12-02T17:15:49,587 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d3557cbe014d2e8cc9a49ae8d70899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733159747793 2024-12-02T17:15:49,597 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#323 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:49,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-02T17:15:49,598 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/384be27ace944dafbe2147c92ba2afa5 is 50, key is test_row_0/C:col10/1733159748458/Put/seqid=0 2024-12-02T17:15:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,601 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/44260484bd284737a774d652b1aea9ab is 50, key is test_row_0/A:col10/1733159748510/Put/seqid=0 2024-12-02T17:15:49,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742197_1373 (size=13323) 2024-12-02T17:15:49,634 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/384be27ace944dafbe2147c92ba2afa5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/384be27ace944dafbe2147c92ba2afa5 2024-12-02T17:15:49,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742198_1374 (size=12301) 2024-12-02T17:15:49,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/44260484bd284737a774d652b1aea9ab 2024-12-02T17:15:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:49,672 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 384be27ace944dafbe2147c92ba2afa5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:49,672 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:49,672 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159749516; duration=0sec 2024-12-02T17:15:49,672 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:49,672 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:49,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/59ec426fc2a345a8966c2acc467fb212 is 50, key is test_row_0/B:col10/1733159748510/Put/seqid=0 2024-12-02T17:15:49,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742199_1375 (size=12301) 2024-12-02T17:15:49,731 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/59ec426fc2a345a8966c2acc467fb212 2024-12-02T17:15:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/95137ec3731f4e7eaeffa41edad25009 is 50, key is test_row_0/C:col10/1733159748510/Put/seqid=0 2024-12-02T17:15:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T17:15:49,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159809761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55752 deadline: 1733159809767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,773 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:49,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159749753, max timestamp=1733159749770Num uniques 859; 2024-12-02T17:15:49,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:49,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742200_1376 (size=12301) 2024-12-02T17:15:49,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/95137ec3731f4e7eaeffa41edad25009 2024-12-02T17:15:49,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159749753, max timestamp=1733159749770Num uniques 859; 2024-12-02T17:15:49,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:49,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159749753, max timestamp=1733159749770Num uniques 859; 2024-12-02T17:15:49,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:15:49,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/44260484bd284737a774d652b1aea9ab as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/44260484bd284737a774d652b1aea9ab 2024-12-02T17:15:49,797 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/44260484bd284737a774d652b1aea9ab, entries=150, sequenceid=526, filesize=12.0 K 2024-12-02T17:15:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/59ec426fc2a345a8966c2acc467fb212 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/59ec426fc2a345a8966c2acc467fb212 2024-12-02T17:15:49,802 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/59ec426fc2a345a8966c2acc467fb212, entries=150, sequenceid=526, filesize=12.0 K 2024-12-02T17:15:49,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/95137ec3731f4e7eaeffa41edad25009 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95137ec3731f4e7eaeffa41edad25009 2024-12-02T17:15:49,807 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95137ec3731f4e7eaeffa41edad25009, entries=150, sequenceid=526, filesize=12.0 K 2024-12-02T17:15:49,808 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=73.80 KB/75570 for cfac2600ff5eef41b88bb4f556e87599 in 208ms, sequenceid=526, compaction requested=false 2024-12-02T17:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:49,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-02T17:15:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-02T17:15:49,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=80.51 KB heapSize=220.92 KB 2024-12-02T17:15:49,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:49,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:49,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:49,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:49,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:49,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:15:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:49,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-02T17:15:49,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 676 msec 2024-12-02T17:15:49,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/5501e3b3e34a41e78c19e31f6595c73b is 50, key is test_row_0/A:col10/1733159749770/Put/seqid=0 2024-12-02T17:15:49,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 686 msec 2024-12-02T17:15:49,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742201_1377 (size=14741) 2024-12-02T17:15:49,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/5501e3b3e34a41e78c19e31f6595c73b 2024-12-02T17:15:49,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/1819f535967f45a9a54464311681756d is 50, key is test_row_0/B:col10/1733159749770/Put/seqid=0 2024-12-02T17:15:49,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742202_1378 (size=12301) 2024-12-02T17:15:49,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/1819f535967f45a9a54464311681756d 2024-12-02T17:15:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159809896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159809905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159809909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:49,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/a0d5fe53456944eabfbdb549bc6cc5f5 is 50, key is test_row_0/C:col10/1733159749770/Put/seqid=0 2024-12-02T17:15:49,981 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/75c0e0001df6496f9f74240bbe838dee as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/75c0e0001df6496f9f74240bbe838dee 2024-12-02T17:15:49,987 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 75c0e0001df6496f9f74240bbe838dee(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:49,987 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:49,987 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159749516; duration=0sec 2024-12-02T17:15:49,988 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:49,988 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:50,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742203_1379 (size=12301) 2024-12-02T17:15:50,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/a0d5fe53456944eabfbdb549bc6cc5f5 2024-12-02T17:15:50,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159810017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159810017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/5501e3b3e34a41e78c19e31f6595c73b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/5501e3b3e34a41e78c19e31f6595c73b 2024-12-02T17:15:50,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159810032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/5501e3b3e34a41e78c19e31f6595c73b, entries=200, sequenceid=555, filesize=14.4 K 2024-12-02T17:15:50,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/1819f535967f45a9a54464311681756d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/1819f535967f45a9a54464311681756d 2024-12-02T17:15:50,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/1819f535967f45a9a54464311681756d, entries=150, sequenceid=555, filesize=12.0 K 2024-12-02T17:15:50,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/a0d5fe53456944eabfbdb549bc6cc5f5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/a0d5fe53456944eabfbdb549bc6cc5f5 2024-12-02T17:15:50,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/a0d5fe53456944eabfbdb549bc6cc5f5, entries=150, sequenceid=555, filesize=12.0 K 2024-12-02T17:15:50,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~220.88 KB/226176, currentSize=100.63 KB/103050 for cfac2600ff5eef41b88bb4f556e87599 in 250ms, sequenceid=555, compaction requested=true 2024-12-02T17:15:50,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:50,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:50,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:50,061 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:50,061 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:50,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:50,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:50,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:50,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:50,062 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:50,062 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:50,062 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:50,062 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:50,062 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:50,062 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:50,062 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/75c0e0001df6496f9f74240bbe838dee, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/59ec426fc2a345a8966c2acc467fb212, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/1819f535967f45a9a54464311681756d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=37.0 K 2024-12-02T17:15:50,062 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2f2678bbb225468b95b597de023cc4a2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/44260484bd284737a774d652b1aea9ab, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/5501e3b3e34a41e78c19e31f6595c73b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=39.4 K 2024-12-02T17:15:50,063 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c0e0001df6496f9f74240bbe838dee, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733159747793 2024-12-02T17:15:50,063 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f2678bbb225468b95b597de023cc4a2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733159747793 2024-12-02T17:15:50,063 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44260484bd284737a774d652b1aea9ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733159748496 2024-12-02T17:15:50,063 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 59ec426fc2a345a8966c2acc467fb212, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733159748496 2024-12-02T17:15:50,063 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5501e3b3e34a41e78c19e31f6595c73b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733159749770 2024-12-02T17:15:50,063 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1819f535967f45a9a54464311681756d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733159749770 2024-12-02T17:15:50,079 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:50,079 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/6191aa7e94874ec2b49b5bc9a3a29503 is 50, key is test_row_0/A:col10/1733159749770/Put/seqid=0 2024-12-02T17:15:50,092 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:50,092 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/41b0a7cd9526412eb15f9b4b940cfff0 is 50, key is test_row_0/B:col10/1733159749770/Put/seqid=0 2024-12-02T17:15:50,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742204_1380 (size=13425) 2024-12-02T17:15:50,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742205_1381 (size=13425) 2024-12-02T17:15:50,202 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/41b0a7cd9526412eb15f9b4b940cfff0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/41b0a7cd9526412eb15f9b4b940cfff0 2024-12-02T17:15:50,217 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 41b0a7cd9526412eb15f9b4b940cfff0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:50,217 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:50,217 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=13, startTime=1733159750061; duration=0sec 2024-12-02T17:15:50,217 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:50,217 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:50,217 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:15:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:50,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:15:50,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:50,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:50,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:50,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:50,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:50,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:50,239 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:15:50,239 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:50,239 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:50,240 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/384be27ace944dafbe2147c92ba2afa5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95137ec3731f4e7eaeffa41edad25009, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/a0d5fe53456944eabfbdb549bc6cc5f5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=37.0 K 2024-12-02T17:15:50,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/835e6e741ae94b00aec0af2f82bd9e51 is 50, key is test_row_0/A:col10/1733159749896/Put/seqid=0 2024-12-02T17:15:50,247 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 384be27ace944dafbe2147c92ba2afa5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733159747793 2024-12-02T17:15:50,247 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 95137ec3731f4e7eaeffa41edad25009, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733159748496 2024-12-02T17:15:50,248 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d5fe53456944eabfbdb549bc6cc5f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733159749770 2024-12-02T17:15:50,265 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:50,266 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/99dd887d01ac4dd1bba20164d0b03c87 is 50, key is test_row_0/C:col10/1733159749770/Put/seqid=0 2024-12-02T17:15:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-02T17:15:50,273 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:50,274 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:15:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-02T17:15:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T17:15:50,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:15:50,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:15:50,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:15:50,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159810274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742206_1382 (size=14741) 2024-12-02T17:15:50,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/835e6e741ae94b00aec0af2f82bd9e51 2024-12-02T17:15:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159810286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159810286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742207_1383 (size=13425) 2024-12-02T17:15:50,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/42b3823ea3c34684b19f7f7af094e616 is 50, key is test_row_0/B:col10/1733159749896/Put/seqid=0 2024-12-02T17:15:50,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742208_1384 (size=12301) 2024-12-02T17:15:50,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/42b3823ea3c34684b19f7f7af094e616 2024-12-02T17:15:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T17:15:50,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/38b835b3fe8249dd85d8605806f994dc is 50, key is test_row_0/C:col10/1733159749896/Put/seqid=0 2024-12-02T17:15:50,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159810408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159810413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159810413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T17:15:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:50,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:50,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:15:50,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742209_1385 (size=12301) 2024-12-02T17:15:50,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/38b835b3fe8249dd85d8605806f994dc 2024-12-02T17:15:50,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/835e6e741ae94b00aec0af2f82bd9e51 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/835e6e741ae94b00aec0af2f82bd9e51 2024-12-02T17:15:50,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/835e6e741ae94b00aec0af2f82bd9e51, entries=200, sequenceid=576, filesize=14.4 K 2024-12-02T17:15:50,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/42b3823ea3c34684b19f7f7af094e616 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/42b3823ea3c34684b19f7f7af094e616 2024-12-02T17:15:50,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/42b3823ea3c34684b19f7f7af094e616, entries=150, sequenceid=576, filesize=12.0 K 2024-12-02T17:15:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/38b835b3fe8249dd85d8605806f994dc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/38b835b3fe8249dd85d8605806f994dc 2024-12-02T17:15:50,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/38b835b3fe8249dd85d8605806f994dc, entries=150, sequenceid=576, filesize=12.0 K 2024-12-02T17:15:50,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cfac2600ff5eef41b88bb4f556e87599 in 308ms, sequenceid=576, compaction requested=false 2024-12-02T17:15:50,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:50,572 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/6191aa7e94874ec2b49b5bc9a3a29503 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6191aa7e94874ec2b49b5bc9a3a29503 2024-12-02T17:15:50,586 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into 6191aa7e94874ec2b49b5bc9a3a29503(size=13.1 K), total size for store is 27.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:50,586 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:50,586 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=13, startTime=1733159750060; duration=0sec 2024-12-02T17:15:50,586 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:50,586 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:50,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-02T17:15:50,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:50,596 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:15:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:50,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T17:15:50,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/84a9e49e28b5429ba3d7cc69bef41122 is 50, key is test_row_0/A:col10/1733159750252/Put/seqid=0 2024-12-02T17:15:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:50,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. as already flushing 2024-12-02T17:15:50,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742210_1386 (size=12301) 2024-12-02T17:15:50,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159810667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159810674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159810678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,712 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/99dd887d01ac4dd1bba20164d0b03c87 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/99dd887d01ac4dd1bba20164d0b03c87 2024-12-02T17:15:50,724 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 99dd887d01ac4dd1bba20164d0b03c87(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:50,724 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:50,724 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=13, startTime=1733159750061; duration=0sec 2024-12-02T17:15:50,724 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:50,724 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159810787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159810787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159810797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:50,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:50,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T17:15:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159810996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159810997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159811018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,034 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/84a9e49e28b5429ba3d7cc69bef41122 2024-12-02T17:15:51,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55700 deadline: 1733159811037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,045 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in cfac2600ff5eef41b88bb4f556e87599 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18364 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:15:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8c5d297d14cb45089c86d93487cf76fc is 50, key is test_row_0/B:col10/1733159750252/Put/seqid=0 2024-12-02T17:15:51,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742211_1387 (size=12301) 2024-12-02T17:15:51,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8c5d297d14cb45089c86d93487cf76fc 2024-12-02T17:15:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d9c62c8b8d9844abaefa6578b1c65840 is 50, key is test_row_0/C:col10/1733159750252/Put/seqid=0 2024-12-02T17:15:51,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742212_1388 (size=12301) 2024-12-02T17:15:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55736 deadline: 1733159811307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55664 deadline: 1733159811307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:15:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55688 deadline: 1733159811327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:15:51,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:15:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T17:15:51,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d9c62c8b8d9844abaefa6578b1c65840 2024-12-02T17:15:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/84a9e49e28b5429ba3d7cc69bef41122 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/84a9e49e28b5429ba3d7cc69bef41122 2024-12-02T17:15:51,574 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/84a9e49e28b5429ba3d7cc69bef41122, entries=150, sequenceid=590, filesize=12.0 K 2024-12-02T17:15:51,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8c5d297d14cb45089c86d93487cf76fc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8c5d297d14cb45089c86d93487cf76fc 2024-12-02T17:15:51,583 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8c5d297d14cb45089c86d93487cf76fc, entries=150, sequenceid=590, filesize=12.0 K 2024-12-02T17:15:51,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/d9c62c8b8d9844abaefa6578b1c65840 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d9c62c8b8d9844abaefa6578b1c65840 2024-12-02T17:15:51,588 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d9c62c8b8d9844abaefa6578b1c65840, entries=150, sequenceid=590, filesize=12.0 K 2024-12-02T17:15:51,589 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cfac2600ff5eef41b88bb4f556e87599 in 993ms, sequenceid=590, compaction requested=true 2024-12-02T17:15:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:51,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-02T17:15:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-02T17:15:51,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-02T17:15:51,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3200 sec 2024-12-02T17:15:51,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3290 sec 2024-12-02T17:15:51,787 INFO [Thread-1297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1297. 2024-12-02T17:15:51,787 DEBUG [Thread-1297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,787 DEBUG [Thread-1297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,787 DEBUG [Thread-1297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,788 INFO [Thread-1291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1291. 2024-12-02T17:15:51,788 DEBUG [Thread-1291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,788 DEBUG [Thread-1291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,788 DEBUG [Thread-1291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,793 INFO [Thread-1299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1299. 2024-12-02T17:15:51,793 DEBUG [Thread-1299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,793 DEBUG [Thread-1299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,793 DEBUG [Thread-1299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,794 INFO [Thread-1295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1295. 2024-12-02T17:15:51,794 DEBUG [Thread-1295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,794 DEBUG [Thread-1295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,794 DEBUG [Thread-1295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,796 INFO [Thread-1293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1293. 2024-12-02T17:15:51,796 DEBUG [Thread-1293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,796 DEBUG [Thread-1293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,796 DEBUG [Thread-1293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:15:51,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:15:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:15:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:15:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:15:51,823 INFO [Thread-1282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1282. 2024-12-02T17:15:51,823 DEBUG [Thread-1282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,823 DEBUG [Thread-1282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,823 DEBUG [Thread-1282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,829 INFO [Thread-1288 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1288. 2024-12-02T17:15:51,829 DEBUG [Thread-1288 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,829 DEBUG [Thread-1288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,830 DEBUG [Thread-1288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/e18e4211f537489aa1afd1bf188647d7 is 50, key is test_row_0/A:col10/1733159751819/Put/seqid=0 2024-12-02T17:15:51,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742213_1389 (size=12301) 2024-12-02T17:15:51,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/e18e4211f537489aa1afd1bf188647d7 2024-12-02T17:15:51,856 INFO [Thread-1286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1286. 2024-12-02T17:15:51,856 DEBUG [Thread-1286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:51,856 DEBUG [Thread-1286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,856 DEBUG [Thread-1286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:51,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:15:51,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8eb64e9c078441e5b26ce68088977dcd is 50, key is test_row_0/B:col10/1733159751819/Put/seqid=0 2024-12-02T17:15:51,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742214_1390 (size=12301) 2024-12-02T17:15:52,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8eb64e9c078441e5b26ce68088977dcd 2024-12-02T17:15:52,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/34846bee8cfb40409e21fdfc54dc8e2f is 50, key is test_row_0/C:col10/1733159751819/Put/seqid=0 2024-12-02T17:15:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742215_1391 (size=12301) 2024-12-02T17:15:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-02T17:15:52,441 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:15:52,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/34846bee8cfb40409e21fdfc54dc8e2f 2024-12-02T17:15:52,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/e18e4211f537489aa1afd1bf188647d7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/e18e4211f537489aa1afd1bf188647d7 2024-12-02T17:15:52,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/e18e4211f537489aa1afd1bf188647d7, entries=150, sequenceid=611, filesize=12.0 K 2024-12-02T17:15:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/8eb64e9c078441e5b26ce68088977dcd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8eb64e9c078441e5b26ce68088977dcd 2024-12-02T17:15:52,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8eb64e9c078441e5b26ce68088977dcd, entries=150, sequenceid=611, filesize=12.0 K 2024-12-02T17:15:52,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/34846bee8cfb40409e21fdfc54dc8e2f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/34846bee8cfb40409e21fdfc54dc8e2f 2024-12-02T17:15:52,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/34846bee8cfb40409e21fdfc54dc8e2f, entries=150, sequenceid=611, filesize=12.0 K 2024-12-02T17:15:52,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for cfac2600ff5eef41b88bb4f556e87599 in 884ms, sequenceid=611, compaction requested=true 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:15:52,705 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfac2600ff5eef41b88bb4f556e87599:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:15:52,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:52,705 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:52,706 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:52,706 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:52,706 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/B is initiating minor compaction (all files) 2024-12-02T17:15:52,706 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/A is initiating minor compaction (all files) 2024-12-02T17:15:52,706 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/B in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:52,706 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/A in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:52,707 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6191aa7e94874ec2b49b5bc9a3a29503, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/835e6e741ae94b00aec0af2f82bd9e51, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/84a9e49e28b5429ba3d7cc69bef41122, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/e18e4211f537489aa1afd1bf188647d7] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=51.5 K 2024-12-02T17:15:52,707 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/41b0a7cd9526412eb15f9b4b940cfff0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/42b3823ea3c34684b19f7f7af094e616, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8c5d297d14cb45089c86d93487cf76fc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8eb64e9c078441e5b26ce68088977dcd] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=49.1 K 2024-12-02T17:15:52,707 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6191aa7e94874ec2b49b5bc9a3a29503, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733159749770 2024-12-02T17:15:52,707 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b0a7cd9526412eb15f9b4b940cfff0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733159749770 2024-12-02T17:15:52,707 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 42b3823ea3c34684b19f7f7af094e616, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733159749894 2024-12-02T17:15:52,707 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835e6e741ae94b00aec0af2f82bd9e51, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733159749894 2024-12-02T17:15:52,707 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a9e49e28b5429ba3d7cc69bef41122, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1733159750252 2024-12-02T17:15:52,707 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c5d297d14cb45089c86d93487cf76fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1733159750252 2024-12-02T17:15:52,708 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb64e9c078441e5b26ce68088977dcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733159750651 2024-12-02T17:15:52,708 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e18e4211f537489aa1afd1bf188647d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733159750651 2024-12-02T17:15:52,715 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:52,715 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#A#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:52,715 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/6419467f96ae4d86855f95d03752df8e is 50, key is test_row_0/B:col10/1733159751819/Put/seqid=0 2024-12-02T17:15:52,716 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/bf9edb632de84efb91c919244d985e27 is 50, key is test_row_0/A:col10/1733159751819/Put/seqid=0 2024-12-02T17:15:52,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742216_1392 (size=13561) 2024-12-02T17:15:52,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742217_1393 (size=13561) 2024-12-02T17:15:53,123 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/6419467f96ae4d86855f95d03752df8e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6419467f96ae4d86855f95d03752df8e 2024-12-02T17:15:53,123 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/bf9edb632de84efb91c919244d985e27 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/bf9edb632de84efb91c919244d985e27 2024-12-02T17:15:53,128 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/B of cfac2600ff5eef41b88bb4f556e87599 into 6419467f96ae4d86855f95d03752df8e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:53,128 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/A of cfac2600ff5eef41b88bb4f556e87599 into bf9edb632de84efb91c919244d985e27(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:53,128 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/B, priority=12, startTime=1733159752705; duration=0sec 2024-12-02T17:15:53,128 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/A, priority=12, startTime=1733159752705; duration=0sec 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:B 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:A 2024-12-02T17:15:53,128 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:15:53,130 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:15:53,130 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): cfac2600ff5eef41b88bb4f556e87599/C is initiating minor compaction (all files) 2024-12-02T17:15:53,130 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfac2600ff5eef41b88bb4f556e87599/C in TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:15:53,130 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/99dd887d01ac4dd1bba20164d0b03c87, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/38b835b3fe8249dd85d8605806f994dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d9c62c8b8d9844abaefa6578b1c65840, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/34846bee8cfb40409e21fdfc54dc8e2f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp, totalSize=49.1 K 2024-12-02T17:15:53,130 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 99dd887d01ac4dd1bba20164d0b03c87, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733159749770 2024-12-02T17:15:53,131 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b835b3fe8249dd85d8605806f994dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733159749894 2024-12-02T17:15:53,131 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c62c8b8d9844abaefa6578b1c65840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1733159750252 2024-12-02T17:15:53,131 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 34846bee8cfb40409e21fdfc54dc8e2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733159750651 2024-12-02T17:15:53,140 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfac2600ff5eef41b88bb4f556e87599#C#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:15:53,140 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/83c86c43bb92492980bb5fe538be1224 is 50, key is test_row_0/C:col10/1733159751819/Put/seqid=0 2024-12-02T17:15:53,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742218_1394 (size=13561) 2024-12-02T17:15:53,558 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/83c86c43bb92492980bb5fe538be1224 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83c86c43bb92492980bb5fe538be1224 2024-12-02T17:15:53,564 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfac2600ff5eef41b88bb4f556e87599/C of cfac2600ff5eef41b88bb4f556e87599 into 83c86c43bb92492980bb5fe538be1224(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:15:53,564 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfac2600ff5eef41b88bb4f556e87599: 2024-12-02T17:15:53,565 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., storeName=cfac2600ff5eef41b88bb4f556e87599/C, priority=12, startTime=1733159752705; duration=0sec 2024-12-02T17:15:53,565 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:15:53,565 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfac2600ff5eef41b88bb4f556e87599:C 2024-12-02T17:15:54,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T17:15:59,788 INFO [Thread-1280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1280. 2024-12-02T17:15:59,788 DEBUG [Thread-1280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:15:59,788 DEBUG [Thread-1280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:59,788 DEBUG [Thread-1280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:15:59,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:01,081 INFO [Thread-1284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1284. 2024-12-02T17:16:01,081 DEBUG [Thread-1284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:01,081 DEBUG [Thread-1284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:01,081 DEBUG [Thread-1284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:01,081 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T17:16:01,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1789 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1740 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1712 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1804 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1729 2024-12-02T17:16:01,082 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T17:16:01,082 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:16:01,082 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:16:01,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:01,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:01,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:01,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T17:16:01,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:01,096 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159761096"}]},"ts":"1733159761096"} 2024-12-02T17:16:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T17:16:01,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T17:16:01,106 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T17:16:01,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:16:01,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, UNASSIGN}] 2024-12-02T17:16:01,109 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, UNASSIGN 2024-12-02T17:16:01,110 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=cfac2600ff5eef41b88bb4f556e87599, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:01,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, UNASSIGN because future has completed 2024-12-02T17:16:01,129 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:16:01,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T17:16:01,305 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:16:01,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:16:01,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing cfac2600ff5eef41b88bb4f556e87599, disabling compactions & flushes 2024-12-02T17:16:01,305 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:16:01,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:16:01,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. after waiting 0 ms 2024-12-02T17:16:01,305 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:16:01,305 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing cfac2600ff5eef41b88bb4f556e87599 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T17:16:01,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=A 2024-12-02T17:16:01,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:01,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=B 2024-12-02T17:16:01,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:01,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfac2600ff5eef41b88bb4f556e87599, store=C 2024-12-02T17:16:01,306 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:01,320 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/80589dbb151541cba7def78b17e48f0f is 50, key is test_row_0/A:col10/1733159761069/Put/seqid=0 2024-12-02T17:16:01,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742219_1395 (size=9857) 2024-12-02T17:16:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T17:16:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T17:16:01,785 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/80589dbb151541cba7def78b17e48f0f 2024-12-02T17:16:01,815 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/22c7d6a50a1c49c2a2460c575c2f5748 is 50, key is test_row_0/B:col10/1733159761069/Put/seqid=0 2024-12-02T17:16:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742220_1396 (size=9857) 2024-12-02T17:16:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T17:16:02,260 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/22c7d6a50a1c49c2a2460c575c2f5748 2024-12-02T17:16:02,270 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b72d3ca0ca0d48fcaee591aeecc71a97 is 50, key is test_row_0/C:col10/1733159761069/Put/seqid=0 2024-12-02T17:16:02,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742221_1397 (size=9857) 2024-12-02T17:16:02,299 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b72d3ca0ca0d48fcaee591aeecc71a97 2024-12-02T17:16:02,309 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/A/80589dbb151541cba7def78b17e48f0f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/80589dbb151541cba7def78b17e48f0f 2024-12-02T17:16:02,314 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/80589dbb151541cba7def78b17e48f0f, entries=100, sequenceid=621, filesize=9.6 K 2024-12-02T17:16:02,316 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/B/22c7d6a50a1c49c2a2460c575c2f5748 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/22c7d6a50a1c49c2a2460c575c2f5748 2024-12-02T17:16:02,320 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/22c7d6a50a1c49c2a2460c575c2f5748, entries=100, sequenceid=621, filesize=9.6 K 2024-12-02T17:16:02,329 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/.tmp/C/b72d3ca0ca0d48fcaee591aeecc71a97 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b72d3ca0ca0d48fcaee591aeecc71a97 2024-12-02T17:16:02,336 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b72d3ca0ca0d48fcaee591aeecc71a97, entries=100, sequenceid=621, filesize=9.6 K 2024-12-02T17:16:02,337 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cfac2600ff5eef41b88bb4f556e87599 in 1032ms, sequenceid=621, compaction requested=false 2024-12-02T17:16:02,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/25dfb884632448d7b76d41425b3895e3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/7698952280ab46a7a7d20889d2a13fe1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d7a57066418c48fb80d5091854a4ef17, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/9718de60f1df4bc7876135b96f27794d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1beb36860dd047a3be96bddfcf87e2c7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6504d26915c14c22abcc2589b2583e5d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/b56be8a4444e4c6bae7226357a2c77ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1e5a331762ab49d5ba3462bd2e6e7b75, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/041d7abbe2664b6683a30e738f014832, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/66a5bd51208b44ba969fd081e1537f66, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2b5ce353710b499d85c0c23747a80a2f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/4684d4975f2641ea9df076ccc1eae0a3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/61ff89374b484414bbc0f1cc1d3bacc4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/29b154a9a63140adac7555c742aa138f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8dab7d824fcf4631b7b7ab6ebc201213, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f21fabb90c9041b3979383fdbea0fcfa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/96f4a75fe5774d2ba7452fd0f91e445f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/aa4ce143a4d447b4a910ed8416564e77, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/dae71028e5fc425eba5106399e09b5ae, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f86f63af5787472c90b4fc4cf74e15b8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8489dfd9c7a344b59eda8c911c6d4d69, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f621af18d7984cb4bcbdee517b4e3b01, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d47e7211a0704e598a94b2dc5a76aead, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/c43e8bb76746466892ac3276f8ce871b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/cef7f7fbda3148bc92446bc14a2959ec, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/65dd4d430a0947b4bb6d2a06788d1cf4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/81a87451899d4581a893a1331b3dcc35, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/ebbbf8be8f6c47f68966360cf9cbac1c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/da7fcca850414d9d82fbd4cd61f7fefe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2f2678bbb225468b95b597de023cc4a2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2a6230c76dfc4ebe800d35f1cce610f5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/44260484bd284737a774d652b1aea9ab, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/5501e3b3e34a41e78c19e31f6595c73b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6191aa7e94874ec2b49b5bc9a3a29503, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/835e6e741ae94b00aec0af2f82bd9e51, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/84a9e49e28b5429ba3d7cc69bef41122, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/e18e4211f537489aa1afd1bf188647d7] to archive 2024-12-02T17:16:02,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:02,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/25dfb884632448d7b76d41425b3895e3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/25dfb884632448d7b76d41425b3895e3 2024-12-02T17:16:02,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/7698952280ab46a7a7d20889d2a13fe1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/7698952280ab46a7a7d20889d2a13fe1 2024-12-02T17:16:02,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d7a57066418c48fb80d5091854a4ef17 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d7a57066418c48fb80d5091854a4ef17 2024-12-02T17:16:02,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/9718de60f1df4bc7876135b96f27794d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/9718de60f1df4bc7876135b96f27794d 2024-12-02T17:16:02,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1beb36860dd047a3be96bddfcf87e2c7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1beb36860dd047a3be96bddfcf87e2c7 2024-12-02T17:16:02,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6504d26915c14c22abcc2589b2583e5d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6504d26915c14c22abcc2589b2583e5d 2024-12-02T17:16:02,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/b56be8a4444e4c6bae7226357a2c77ea to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/b56be8a4444e4c6bae7226357a2c77ea 2024-12-02T17:16:02,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1e5a331762ab49d5ba3462bd2e6e7b75 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/1e5a331762ab49d5ba3462bd2e6e7b75 2024-12-02T17:16:02,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/041d7abbe2664b6683a30e738f014832 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/041d7abbe2664b6683a30e738f014832 2024-12-02T17:16:02,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/66a5bd51208b44ba969fd081e1537f66 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/66a5bd51208b44ba969fd081e1537f66 2024-12-02T17:16:02,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2b5ce353710b499d85c0c23747a80a2f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2b5ce353710b499d85c0c23747a80a2f 2024-12-02T17:16:02,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/4684d4975f2641ea9df076ccc1eae0a3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/4684d4975f2641ea9df076ccc1eae0a3 2024-12-02T17:16:02,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/61ff89374b484414bbc0f1cc1d3bacc4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/61ff89374b484414bbc0f1cc1d3bacc4 2024-12-02T17:16:02,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/29b154a9a63140adac7555c742aa138f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/29b154a9a63140adac7555c742aa138f 2024-12-02T17:16:02,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8dab7d824fcf4631b7b7ab6ebc201213 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8dab7d824fcf4631b7b7ab6ebc201213 2024-12-02T17:16:02,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f21fabb90c9041b3979383fdbea0fcfa to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f21fabb90c9041b3979383fdbea0fcfa 2024-12-02T17:16:02,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/96f4a75fe5774d2ba7452fd0f91e445f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/96f4a75fe5774d2ba7452fd0f91e445f 2024-12-02T17:16:02,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/aa4ce143a4d447b4a910ed8416564e77 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/aa4ce143a4d447b4a910ed8416564e77 2024-12-02T17:16:02,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/dae71028e5fc425eba5106399e09b5ae to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/dae71028e5fc425eba5106399e09b5ae 2024-12-02T17:16:02,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f86f63af5787472c90b4fc4cf74e15b8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f86f63af5787472c90b4fc4cf74e15b8 2024-12-02T17:16:02,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8489dfd9c7a344b59eda8c911c6d4d69 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/8489dfd9c7a344b59eda8c911c6d4d69 2024-12-02T17:16:02,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f621af18d7984cb4bcbdee517b4e3b01 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/f621af18d7984cb4bcbdee517b4e3b01 2024-12-02T17:16:02,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d47e7211a0704e598a94b2dc5a76aead to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/d47e7211a0704e598a94b2dc5a76aead 2024-12-02T17:16:02,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/c43e8bb76746466892ac3276f8ce871b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/c43e8bb76746466892ac3276f8ce871b 2024-12-02T17:16:02,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/cef7f7fbda3148bc92446bc14a2959ec to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/cef7f7fbda3148bc92446bc14a2959ec 2024-12-02T17:16:02,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/65dd4d430a0947b4bb6d2a06788d1cf4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/65dd4d430a0947b4bb6d2a06788d1cf4 2024-12-02T17:16:02,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/81a87451899d4581a893a1331b3dcc35 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/81a87451899d4581a893a1331b3dcc35 2024-12-02T17:16:02,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/ebbbf8be8f6c47f68966360cf9cbac1c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/ebbbf8be8f6c47f68966360cf9cbac1c 2024-12-02T17:16:02,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/da7fcca850414d9d82fbd4cd61f7fefe to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/da7fcca850414d9d82fbd4cd61f7fefe 2024-12-02T17:16:02,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2f2678bbb225468b95b597de023cc4a2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2f2678bbb225468b95b597de023cc4a2 2024-12-02T17:16:02,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2a6230c76dfc4ebe800d35f1cce610f5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/2a6230c76dfc4ebe800d35f1cce610f5 2024-12-02T17:16:02,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/44260484bd284737a774d652b1aea9ab to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/44260484bd284737a774d652b1aea9ab 2024-12-02T17:16:02,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/5501e3b3e34a41e78c19e31f6595c73b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/5501e3b3e34a41e78c19e31f6595c73b 2024-12-02T17:16:02,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6191aa7e94874ec2b49b5bc9a3a29503 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/6191aa7e94874ec2b49b5bc9a3a29503 2024-12-02T17:16:02,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/835e6e741ae94b00aec0af2f82bd9e51 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/835e6e741ae94b00aec0af2f82bd9e51 2024-12-02T17:16:02,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/84a9e49e28b5429ba3d7cc69bef41122 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/84a9e49e28b5429ba3d7cc69bef41122 2024-12-02T17:16:02,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/e18e4211f537489aa1afd1bf188647d7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/e18e4211f537489aa1afd1bf188647d7 2024-12-02T17:16:02,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c3e2c85f428545489263bd13f1300329, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/fa6171d9cb5740f0874e8003dab2687d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/27fbce7191764784a944b8d896668e1d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/106420c9deb0442fafecc8c804bd7d74, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9611188c0a304f3cb98418a80f7002a2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/47d2490a890a4156b20dd25c6843965f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/250111df010c40939ba2bd895145913b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f8f151132dd14be987948ddc188e11da, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f6b64bfde5ee4c8785f871dedf4a9c86, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7c3fa51b2ea84226b7f1ee3481532584, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c76399da9e3a49b18197184746ee9817, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8245698f24a14d46bf8e62e51f4b677a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/bc92f0305ad74a60bfd6d5b5725e287d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e7a6442c3f7a42d5abbf687908216dfc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/ab98336b750b417a864cbbb795c02d40, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9777756af96c47a5a8576638a7bae237, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4c75362e3dca42c19c582cbc41951f34, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/2d4098fd0f6049eca984adfe141f5805, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/24ea937cc3e14f23bac157f149faef17, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/87ec764f096942889e39394ff9525005, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/276ddb7534a4417d97476249eb0dba9c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4d90cf336a3645ba9411b14e43a2076b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/985433b48c434636a6a8b81b77e99d85, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/be1d72c7fb904c67a42e855695c291fa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7779a71f56394db9b8c493b6822bad94, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/67fd6413a79c4db69503c56d294605a0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/71ca618d19d6487d9eca9776a07e7e3a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6970d6ad097b4b7b97b7bb90eb56f906, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e2ac5b93df0c49b9a04679a94d7839e0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/75c0e0001df6496f9f74240bbe838dee, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/373f3d775e224f9fab99ae359c009714, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/59ec426fc2a345a8966c2acc467fb212, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/41b0a7cd9526412eb15f9b4b940cfff0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/1819f535967f45a9a54464311681756d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/42b3823ea3c34684b19f7f7af094e616, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8c5d297d14cb45089c86d93487cf76fc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8eb64e9c078441e5b26ce68088977dcd] to archive 2024-12-02T17:16:02,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:02,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c3e2c85f428545489263bd13f1300329 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c3e2c85f428545489263bd13f1300329 2024-12-02T17:16:02,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/fa6171d9cb5740f0874e8003dab2687d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/fa6171d9cb5740f0874e8003dab2687d 2024-12-02T17:16:02,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/27fbce7191764784a944b8d896668e1d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/27fbce7191764784a944b8d896668e1d 2024-12-02T17:16:02,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/106420c9deb0442fafecc8c804bd7d74 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/106420c9deb0442fafecc8c804bd7d74 2024-12-02T17:16:02,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9611188c0a304f3cb98418a80f7002a2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9611188c0a304f3cb98418a80f7002a2 2024-12-02T17:16:02,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/47d2490a890a4156b20dd25c6843965f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/47d2490a890a4156b20dd25c6843965f 2024-12-02T17:16:02,549 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/250111df010c40939ba2bd895145913b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/250111df010c40939ba2bd895145913b 2024-12-02T17:16:02,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f8f151132dd14be987948ddc188e11da to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f8f151132dd14be987948ddc188e11da 2024-12-02T17:16:02,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f6b64bfde5ee4c8785f871dedf4a9c86 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/f6b64bfde5ee4c8785f871dedf4a9c86 2024-12-02T17:16:02,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7c3fa51b2ea84226b7f1ee3481532584 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7c3fa51b2ea84226b7f1ee3481532584 2024-12-02T17:16:02,554 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c76399da9e3a49b18197184746ee9817 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/c76399da9e3a49b18197184746ee9817 2024-12-02T17:16:02,556 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8245698f24a14d46bf8e62e51f4b677a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8245698f24a14d46bf8e62e51f4b677a 2024-12-02T17:16:02,557 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/bc92f0305ad74a60bfd6d5b5725e287d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/bc92f0305ad74a60bfd6d5b5725e287d 2024-12-02T17:16:02,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e7a6442c3f7a42d5abbf687908216dfc to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e7a6442c3f7a42d5abbf687908216dfc 2024-12-02T17:16:02,560 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/ab98336b750b417a864cbbb795c02d40 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/ab98336b750b417a864cbbb795c02d40 2024-12-02T17:16:02,561 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9777756af96c47a5a8576638a7bae237 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/9777756af96c47a5a8576638a7bae237 2024-12-02T17:16:02,562 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4c75362e3dca42c19c582cbc41951f34 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4c75362e3dca42c19c582cbc41951f34 2024-12-02T17:16:02,565 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/2d4098fd0f6049eca984adfe141f5805 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/2d4098fd0f6049eca984adfe141f5805 2024-12-02T17:16:02,566 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/24ea937cc3e14f23bac157f149faef17 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/24ea937cc3e14f23bac157f149faef17 2024-12-02T17:16:02,568 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/87ec764f096942889e39394ff9525005 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/87ec764f096942889e39394ff9525005 2024-12-02T17:16:02,569 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/276ddb7534a4417d97476249eb0dba9c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/276ddb7534a4417d97476249eb0dba9c 2024-12-02T17:16:02,570 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4d90cf336a3645ba9411b14e43a2076b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/4d90cf336a3645ba9411b14e43a2076b 2024-12-02T17:16:02,572 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/985433b48c434636a6a8b81b77e99d85 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/985433b48c434636a6a8b81b77e99d85 2024-12-02T17:16:02,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/be1d72c7fb904c67a42e855695c291fa to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/be1d72c7fb904c67a42e855695c291fa 2024-12-02T17:16:02,575 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7779a71f56394db9b8c493b6822bad94 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/7779a71f56394db9b8c493b6822bad94 2024-12-02T17:16:02,576 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/67fd6413a79c4db69503c56d294605a0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/67fd6413a79c4db69503c56d294605a0 2024-12-02T17:16:02,577 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/71ca618d19d6487d9eca9776a07e7e3a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/71ca618d19d6487d9eca9776a07e7e3a 2024-12-02T17:16:02,579 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6970d6ad097b4b7b97b7bb90eb56f906 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6970d6ad097b4b7b97b7bb90eb56f906 2024-12-02T17:16:02,580 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e2ac5b93df0c49b9a04679a94d7839e0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/e2ac5b93df0c49b9a04679a94d7839e0 2024-12-02T17:16:02,581 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/75c0e0001df6496f9f74240bbe838dee to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/75c0e0001df6496f9f74240bbe838dee 2024-12-02T17:16:02,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/373f3d775e224f9fab99ae359c009714 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/373f3d775e224f9fab99ae359c009714 2024-12-02T17:16:02,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/59ec426fc2a345a8966c2acc467fb212 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/59ec426fc2a345a8966c2acc467fb212 2024-12-02T17:16:02,586 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/41b0a7cd9526412eb15f9b4b940cfff0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/41b0a7cd9526412eb15f9b4b940cfff0 2024-12-02T17:16:02,589 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/1819f535967f45a9a54464311681756d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/1819f535967f45a9a54464311681756d 2024-12-02T17:16:02,590 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/42b3823ea3c34684b19f7f7af094e616 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/42b3823ea3c34684b19f7f7af094e616 2024-12-02T17:16:02,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8c5d297d14cb45089c86d93487cf76fc to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8c5d297d14cb45089c86d93487cf76fc 2024-12-02T17:16:02,593 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8eb64e9c078441e5b26ce68088977dcd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/8eb64e9c078441e5b26ce68088977dcd 2024-12-02T17:16:02,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/82f4086176a64a83958e1903c6605071, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/ce2b8e744dfe4b2ebdf17325e4462447, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/8959a3680baf4b7e9aff6cb429cb435f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d0479bc13ab741d7b3504e8afe6d0f82, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/69e8617604a84aa29999d770fd9cb3ce, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/041ab73ba5434866967f6772917ae71e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/6e9200bb7a5f44059e1f04fe300c97a1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83019e4ef87e48efa583a5d3b77249c8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d8706968490d49669bb303bc08d7aaa8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b5ea2ec327a64be9b6ab4b260d4f022d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/981969d8a6214805866c44e8bd290ed0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/74b21c110e6941a6baf33572328639a3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95ddbd443cde4f92ac02e1d5ef99f632, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d5f97e046d2a413a99cf9f6d7827447c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/609eea962027457881921c4c95c5fdb5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/17bb62b5a3544a72b04ad13215f143be, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/566d7f98dfcf492aaf2f429af2a54d08, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/64bae6e2e47645db9f4c82d8ea11b730, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/9c0ecfc161544be995c8b06404da0ec8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/afe912b1608f48f1b0b3a0fe2b0e7a19, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/bcb12938b62e412e9b4a72a8926aaaff, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/f58f2e753e9148b1a43363ac4499571f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b7a049bab09148fe9b95fc5675b48384, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/3a8897fdf4ca4593b5441d3b886f41e8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/60612c93b06244c6a7a056a8c771fe88, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/077a525ab2134e2098ac29f8cdbb5630, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/4ca3c38e403740dc932d5c928ae53c25, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/c5f9675cfd714c73a4cb241b99e454fe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/71319ee568fc48969abd2e6021b7bb55, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/384be27ace944dafbe2147c92ba2afa5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/52d3557cbe014d2e8cc9a49ae8d70899, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95137ec3731f4e7eaeffa41edad25009, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/99dd887d01ac4dd1bba20164d0b03c87, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/a0d5fe53456944eabfbdb549bc6cc5f5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/38b835b3fe8249dd85d8605806f994dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d9c62c8b8d9844abaefa6578b1c65840, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/34846bee8cfb40409e21fdfc54dc8e2f] to archive 2024-12-02T17:16:02,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:02,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/82f4086176a64a83958e1903c6605071 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/82f4086176a64a83958e1903c6605071 2024-12-02T17:16:02,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/ce2b8e744dfe4b2ebdf17325e4462447 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/ce2b8e744dfe4b2ebdf17325e4462447 2024-12-02T17:16:02,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/8959a3680baf4b7e9aff6cb429cb435f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/8959a3680baf4b7e9aff6cb429cb435f 2024-12-02T17:16:02,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d0479bc13ab741d7b3504e8afe6d0f82 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d0479bc13ab741d7b3504e8afe6d0f82 2024-12-02T17:16:02,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/69e8617604a84aa29999d770fd9cb3ce to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/69e8617604a84aa29999d770fd9cb3ce 2024-12-02T17:16:02,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/041ab73ba5434866967f6772917ae71e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/041ab73ba5434866967f6772917ae71e 2024-12-02T17:16:02,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/6e9200bb7a5f44059e1f04fe300c97a1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/6e9200bb7a5f44059e1f04fe300c97a1 2024-12-02T17:16:02,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83019e4ef87e48efa583a5d3b77249c8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83019e4ef87e48efa583a5d3b77249c8 2024-12-02T17:16:02,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d8706968490d49669bb303bc08d7aaa8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d8706968490d49669bb303bc08d7aaa8 2024-12-02T17:16:02,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b5ea2ec327a64be9b6ab4b260d4f022d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b5ea2ec327a64be9b6ab4b260d4f022d 2024-12-02T17:16:02,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/981969d8a6214805866c44e8bd290ed0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/981969d8a6214805866c44e8bd290ed0 2024-12-02T17:16:02,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/74b21c110e6941a6baf33572328639a3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/74b21c110e6941a6baf33572328639a3 2024-12-02T17:16:02,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95ddbd443cde4f92ac02e1d5ef99f632 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95ddbd443cde4f92ac02e1d5ef99f632 2024-12-02T17:16:02,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d5f97e046d2a413a99cf9f6d7827447c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d5f97e046d2a413a99cf9f6d7827447c 2024-12-02T17:16:02,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/609eea962027457881921c4c95c5fdb5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/609eea962027457881921c4c95c5fdb5 2024-12-02T17:16:02,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/17bb62b5a3544a72b04ad13215f143be to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/17bb62b5a3544a72b04ad13215f143be 2024-12-02T17:16:02,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/566d7f98dfcf492aaf2f429af2a54d08 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/566d7f98dfcf492aaf2f429af2a54d08 2024-12-02T17:16:02,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/64bae6e2e47645db9f4c82d8ea11b730 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/64bae6e2e47645db9f4c82d8ea11b730 2024-12-02T17:16:02,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/9c0ecfc161544be995c8b06404da0ec8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/9c0ecfc161544be995c8b06404da0ec8 2024-12-02T17:16:02,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/afe912b1608f48f1b0b3a0fe2b0e7a19 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/afe912b1608f48f1b0b3a0fe2b0e7a19 2024-12-02T17:16:02,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/bcb12938b62e412e9b4a72a8926aaaff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/bcb12938b62e412e9b4a72a8926aaaff 2024-12-02T17:16:02,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/f58f2e753e9148b1a43363ac4499571f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/f58f2e753e9148b1a43363ac4499571f 2024-12-02T17:16:02,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b7a049bab09148fe9b95fc5675b48384 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b7a049bab09148fe9b95fc5675b48384 2024-12-02T17:16:02,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/3a8897fdf4ca4593b5441d3b886f41e8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/3a8897fdf4ca4593b5441d3b886f41e8 2024-12-02T17:16:02,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/60612c93b06244c6a7a056a8c771fe88 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/60612c93b06244c6a7a056a8c771fe88 2024-12-02T17:16:02,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/077a525ab2134e2098ac29f8cdbb5630 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/077a525ab2134e2098ac29f8cdbb5630 2024-12-02T17:16:02,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/4ca3c38e403740dc932d5c928ae53c25 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/4ca3c38e403740dc932d5c928ae53c25 2024-12-02T17:16:02,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/c5f9675cfd714c73a4cb241b99e454fe to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/c5f9675cfd714c73a4cb241b99e454fe 2024-12-02T17:16:02,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/71319ee568fc48969abd2e6021b7bb55 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/71319ee568fc48969abd2e6021b7bb55 2024-12-02T17:16:02,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/384be27ace944dafbe2147c92ba2afa5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/384be27ace944dafbe2147c92ba2afa5 2024-12-02T17:16:02,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/52d3557cbe014d2e8cc9a49ae8d70899 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/52d3557cbe014d2e8cc9a49ae8d70899 2024-12-02T17:16:02,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95137ec3731f4e7eaeffa41edad25009 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/95137ec3731f4e7eaeffa41edad25009 2024-12-02T17:16:02,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/99dd887d01ac4dd1bba20164d0b03c87 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/99dd887d01ac4dd1bba20164d0b03c87 2024-12-02T17:16:02,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/a0d5fe53456944eabfbdb549bc6cc5f5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/a0d5fe53456944eabfbdb549bc6cc5f5 2024-12-02T17:16:02,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/38b835b3fe8249dd85d8605806f994dc to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/38b835b3fe8249dd85d8605806f994dc 2024-12-02T17:16:02,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d9c62c8b8d9844abaefa6578b1c65840 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/d9c62c8b8d9844abaefa6578b1c65840 2024-12-02T17:16:02,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/34846bee8cfb40409e21fdfc54dc8e2f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/34846bee8cfb40409e21fdfc54dc8e2f 2024-12-02T17:16:02,925 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/recovered.edits/624.seqid, newMaxSeqId=624, maxSeqId=1 2024-12-02T17:16:02,932 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. 2024-12-02T17:16:02,933 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for cfac2600ff5eef41b88bb4f556e87599: Waiting for close lock at 1733159761305Running coprocessor pre-close hooks at 1733159761305Disabling compacts and flushes for region at 1733159761305Disabling writes for close at 1733159761305Obtaining lock to block concurrent updates at 1733159761305Preparing flush snapshotting stores in cfac2600ff5eef41b88bb4f556e87599 at 1733159761305Finished memstore snapshotting TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733159761306 (+1 ms)Flushing stores of TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599. at 1733159761308 (+2 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/A: creating writer at 1733159761308Flushing cfac2600ff5eef41b88bb4f556e87599/A: appending metadata at 1733159761319 (+11 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/A: closing flushed file at 1733159761320 (+1 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/B: creating writer at 1733159761802 (+482 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/B: appending metadata at 1733159761814 (+12 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/B: closing flushed file at 1733159761814Flushing cfac2600ff5eef41b88bb4f556e87599/C: creating writer at 1733159762264 (+450 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/C: appending metadata at 1733159762269 (+5 ms)Flushing cfac2600ff5eef41b88bb4f556e87599/C: closing flushed file at 1733159762270 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6904720a: reopening flushed file at 1733159762308 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ce93660: reopening flushed file at 1733159762314 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bcd77a: reopening flushed file at 1733159762321 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cfac2600ff5eef41b88bb4f556e87599 in 1032ms, sequenceid=621, compaction requested=false at 1733159762337 (+16 ms)Writing region close event to WAL at 1733159762884 (+547 ms)Running coprocessor post-close hooks at 1733159762932 (+48 ms)Closed at 1733159762932 2024-12-02T17:16:02,940 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:16:02,946 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=cfac2600ff5eef41b88bb4f556e87599, regionState=CLOSED 2024-12-02T17:16:02,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:16:02,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-02T17:16:02,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure cfac2600ff5eef41b88bb4f556e87599, server=3b3375d1a925,40723,1733159666211 in 1.8380 sec 2024-12-02T17:16:02,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-02T17:16:02,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfac2600ff5eef41b88bb4f556e87599, UNASSIGN in 1.8780 sec 2024-12-02T17:16:02,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-02T17:16:02,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8830 sec 2024-12-02T17:16:02,993 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159762993"}]},"ts":"1733159762993"} 2024-12-02T17:16:03,004 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T17:16:03,005 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T17:16:03,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9210 sec 2024-12-02T17:16:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-02T17:16:03,248 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:03,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T17:16:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,251 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,251 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T17:16:03,265 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:16:03,302 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/recovered.edits] 2024-12-02T17:16:03,341 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/80589dbb151541cba7def78b17e48f0f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/80589dbb151541cba7def78b17e48f0f 2024-12-02T17:16:03,356 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/bf9edb632de84efb91c919244d985e27 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/A/bf9edb632de84efb91c919244d985e27 2024-12-02T17:16:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T17:16:03,363 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/22c7d6a50a1c49c2a2460c575c2f5748 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/22c7d6a50a1c49c2a2460c575c2f5748 2024-12-02T17:16:03,372 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6419467f96ae4d86855f95d03752df8e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/B/6419467f96ae4d86855f95d03752df8e 2024-12-02T17:16:03,384 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83c86c43bb92492980bb5fe538be1224 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/83c86c43bb92492980bb5fe538be1224 2024-12-02T17:16:03,396 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b72d3ca0ca0d48fcaee591aeecc71a97 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/C/b72d3ca0ca0d48fcaee591aeecc71a97 2024-12-02T17:16:03,406 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/recovered.edits/624.seqid to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599/recovered.edits/624.seqid 2024-12-02T17:16:03,408 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/cfac2600ff5eef41b88bb4f556e87599 2024-12-02T17:16:03,409 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T17:16:03,432 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,445 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T17:16:03,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T17:16:03,457 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,457 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T17:16:03,457 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733159763457"}]},"ts":"9223372036854775807"} 2024-12-02T17:16:03,462 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T17:16:03,462 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cfac2600ff5eef41b88bb4f556e87599, NAME => 'TestAcidGuarantees,,1733159730957.cfac2600ff5eef41b88bb4f556e87599.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T17:16:03,462 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T17:16:03,464 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733159763462"}]},"ts":"9223372036854775807"} 2024-12-02T17:16:03,469 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T17:16:03,470 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 222 msec 2024-12-02T17:16:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-02T17:16:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T17:16:03,573 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:03,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=983 (was 908) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3208 (was 3835) 2024-12-02T17:16:03,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=983, ProcessCount=11, AvailableMemoryMB=3207 2024-12-02T17:16:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:16:03,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:16:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:03,623 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:16:03,624 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:03,624 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-02T17:16:03,625 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:16:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T17:16:03,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742222_1398 (size=1061) 2024-12-02T17:16:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T17:16:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T17:16:04,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:16:04,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742223_1399 (size=53) 2024-12-02T17:16:04,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:16:04,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 213eb7db536967e2cd87656b3cc2847c, disabling compactions & flushes 2024-12-02T17:16:04,166 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. after waiting 0 ms 2024-12-02T17:16:04,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,166 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,166 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 213eb7db536967e2cd87656b3cc2847c: Waiting for close lock at 1733159764166Disabling compacts and flushes for region at 1733159764166Disabling writes for close at 1733159764166Writing region close event to WAL at 1733159764166Closed at 1733159764166 2024-12-02T17:16:04,172 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:16:04,173 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733159764173"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159764173"}]},"ts":"1733159764173"} 2024-12-02T17:16:04,180 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T17:16:04,188 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:16:04,188 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159764188"}]},"ts":"1733159764188"} 2024-12-02T17:16:04,190 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T17:16:04,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, ASSIGN}] 2024-12-02T17:16:04,192 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, ASSIGN 2024-12-02T17:16:04,208 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, ASSIGN; state=OFFLINE, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=false 2024-12-02T17:16:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T17:16:04,360 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:04,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, ASSIGN because future has completed 2024-12-02T17:16:04,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:04,539 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:16:04,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:16:04,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,540 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,561 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,570 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:04,570 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 213eb7db536967e2cd87656b3cc2847c columnFamilyName A 2024-12-02T17:16:04,570 DEBUG [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:04,574 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(327): Store=213eb7db536967e2cd87656b3cc2847c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:04,575 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,581 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:04,581 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 213eb7db536967e2cd87656b3cc2847c columnFamilyName B 2024-12-02T17:16:04,581 DEBUG [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:04,588 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(327): Store=213eb7db536967e2cd87656b3cc2847c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:04,589 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,601 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:04,601 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 213eb7db536967e2cd87656b3cc2847c columnFamilyName C 2024-12-02T17:16:04,601 DEBUG [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:04,604 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(327): Store=213eb7db536967e2cd87656b3cc2847c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:04,608 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,609 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,609 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,611 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,625 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,625 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,626 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:16:04,627 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,640 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:16:04,641 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 213eb7db536967e2cd87656b3cc2847c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73456676, jitterRate=0.09458976984024048}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:16:04,641 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:04,642 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 213eb7db536967e2cd87656b3cc2847c: Running coprocessor pre-open hook at 1733159764540Writing region info on filesystem at 1733159764540Initializing all the Stores at 1733159764553 (+13 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159764553Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159764560 (+7 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159764560Cleaning up temporary data from old regions at 1733159764625 (+65 ms)Running coprocessor post-open hooks at 1733159764641 (+16 ms)Region opened successfully at 1733159764642 (+1 ms) 2024-12-02T17:16:04,645 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., pid=93, masterSystemTime=1733159764518 2024-12-02T17:16:04,647 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,648 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:04,648 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:04,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:16:04,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-02T17:16:04,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 in 286 msec 2024-12-02T17:16:04,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-02T17:16:04,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, ASSIGN in 472 msec 2024-12-02T17:16:04,681 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:16:04,682 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159764681"}]},"ts":"1733159764681"} 2024-12-02T17:16:04,693 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T17:16:04,694 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:16:04,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0770 sec 2024-12-02T17:16:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T17:16:04,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:04,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:04,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:04,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:04,803 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733569c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:04,806 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d4624b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:04,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:04,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:04,828 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:16:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:16:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1449eeab 2024-12-02T17:16:04,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:16:04,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:16:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:16:04,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:16:04,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:04,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742224_1400 (size=1097) 2024-12-02T17:16:05,342 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T17:16:05,342 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T17:16:05,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:16:05,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, REOPEN/MOVE}] 2024-12-02T17:16:05,357 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, REOPEN/MOVE 2024-12-02T17:16:05,357 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:05,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, REOPEN/MOVE because future has completed 2024-12-02T17:16:05,367 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:16:05,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:05,525 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,525 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:16:05,526 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 213eb7db536967e2cd87656b3cc2847c, disabling compactions & flushes 2024-12-02T17:16:05,526 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,526 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,526 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. after waiting 0 ms 2024-12-02T17:16:05,526 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,556 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:16:05,562 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,562 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 213eb7db536967e2cd87656b3cc2847c: Waiting for close lock at 1733159765526Running coprocessor pre-close hooks at 1733159765526Disabling compacts and flushes for region at 1733159765526Disabling writes for close at 1733159765526Writing region close event to WAL at 1733159765526Running coprocessor post-close hooks at 1733159765562 (+36 ms)Closed at 1733159765562 2024-12-02T17:16:05,562 WARN [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 213eb7db536967e2cd87656b3cc2847c to self. 2024-12-02T17:16:05,574 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=CLOSED 2024-12-02T17:16:05,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:16:05,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-02T17:16:05,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 in 214 msec 2024-12-02T17:16:05,587 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, REOPEN/MOVE; state=CLOSED, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=true 2024-12-02T17:16:05,741 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:05,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, REOPEN/MOVE because future has completed 2024-12-02T17:16:05,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:05,755 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T17:16:05,903 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,903 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:16:05,903 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,903 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:16:05,904 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,904 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,907 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,914 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:05,914 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 213eb7db536967e2cd87656b3cc2847c columnFamilyName A 2024-12-02T17:16:05,921 DEBUG [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:05,922 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(327): Store=213eb7db536967e2cd87656b3cc2847c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:05,923 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,925 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:05,925 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 213eb7db536967e2cd87656b3cc2847c columnFamilyName B 2024-12-02T17:16:05,926 DEBUG [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:05,926 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(327): Store=213eb7db536967e2cd87656b3cc2847c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:05,926 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,927 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:05,927 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 213eb7db536967e2cd87656b3cc2847c columnFamilyName C 2024-12-02T17:16:05,927 DEBUG [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:05,928 INFO [StoreOpener-213eb7db536967e2cd87656b3cc2847c-1 {}] regionserver.HStore(327): Store=213eb7db536967e2cd87656b3cc2847c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:05,928 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,928 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,929 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,931 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,932 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,932 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,932 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:16:05,934 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,935 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 213eb7db536967e2cd87656b3cc2847c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61000919, jitterRate=-0.09101547300815582}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:16:05,935 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:05,936 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 213eb7db536967e2cd87656b3cc2847c: Running coprocessor pre-open hook at 1733159765904Writing region info on filesystem at 1733159765904Initializing all the Stores at 1733159765906 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159765906Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159765906Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159765906Cleaning up temporary data from old regions at 1733159765932 (+26 ms)Running coprocessor post-open hooks at 1733159765935 (+3 ms)Region opened successfully at 1733159765936 (+1 ms) 2024-12-02T17:16:05,938 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., pid=98, masterSystemTime=1733159765899 2024-12-02T17:16:05,941 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=OPEN, openSeqNum=5, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:05,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:16:05,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-02T17:16:05,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 in 200 msec 2024-12-02T17:16:05,947 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,947 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:05,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-02T17:16:05,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, REOPEN/MOVE in 591 msec 2024-12-02T17:16:05,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-02T17:16:05,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 604 msec 2024-12-02T17:16:05,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1120 sec 2024-12-02T17:16:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-02T17:16:05,956 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:05,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:05,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:05,965 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:05,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7253443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:05,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:05,969 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53ec48da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:05,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:05,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5765704c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:05,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325aadc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:05,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:05,995 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8638e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:05,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:05,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d54520, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,002 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0ace45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73399706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,017 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e58e1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1339939f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605a9f61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f37f7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,051 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4b5743, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,062 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390f188e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,073 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da9016e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15626b36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b7b919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,113 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efa9435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,117 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@557b3930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:06,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:06,149 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8911b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:06,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:06,154 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13a014bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:06,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:06,169 DEBUG [Thread-1799 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,169 DEBUG [Thread-1792 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-02T17:16:06,171 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:06,171 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:06,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,177 DEBUG [Thread-1796 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T17:16:06,186 DEBUG [Thread-1805 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,197 DEBUG [Thread-1803 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,209 DEBUG [Thread-1794 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,213 DEBUG [Thread-1790 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,218 DEBUG [Thread-1807 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,232 DEBUG [Thread-1788 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,252 DEBUG [Thread-1801 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:06,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:06,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:16:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T17:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:06,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:06,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:06,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T17:16:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159826405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159826409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202833641cc40da4f5ab3b27f4f111c4790_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159766199/Put/seqid=0 2024-12-02T17:16:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159826413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159826421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159826424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T17:16:06,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:06,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742225_1401 (size=12154) 2024-12-02T17:16:06,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T17:16:06,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202833641cc40da4f5ab3b27f4f111c4790_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202833641cc40da4f5ab3b27f4f111c4790_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:06,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/64760b5fd36a49949ff233b38a1f33ea, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:06,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/64760b5fd36a49949ff233b38a1f33ea is 175, key is test_row_0/A:col10/1733159766199/Put/seqid=0 2024-12-02T17:16:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159826529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159826529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159826549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742226_1402 (size=30955) 2024-12-02T17:16:06,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/64760b5fd36a49949ff233b38a1f33ea 2024-12-02T17:16:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159826559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159826561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/8fb3276a11294eb7a376be509f475687 is 50, key is test_row_0/B:col10/1733159766199/Put/seqid=0 2024-12-02T17:16:06,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T17:16:06,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:06,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742227_1403 (size=12001) 2024-12-02T17:16:06,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/8fb3276a11294eb7a376be509f475687 2024-12-02T17:16:06,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/1ccc60f71413402aa4330567a6c7f6fe is 50, key is test_row_0/C:col10/1733159766199/Put/seqid=0 2024-12-02T17:16:06,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742228_1404 (size=12001) 2024-12-02T17:16:06,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159826749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159826749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/1ccc60f71413402aa4330567a6c7f6fe 2024-12-02T17:16:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159826769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159826782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T17:16:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/64760b5fd36a49949ff233b38a1f33ea as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea 2024-12-02T17:16:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T17:16:06,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea, entries=150, sequenceid=16, filesize=30.2 K 2024-12-02T17:16:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:06,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/8fb3276a11294eb7a376be509f475687 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/8fb3276a11294eb7a376be509f475687 2024-12-02T17:16:06,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159826797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:06,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:06,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/8fb3276a11294eb7a376be509f475687, entries=150, sequenceid=16, filesize=11.7 K 2024-12-02T17:16:06,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/1ccc60f71413402aa4330567a6c7f6fe as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ccc60f71413402aa4330567a6c7f6fe 2024-12-02T17:16:06,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ccc60f71413402aa4330567a6c7f6fe, entries=150, sequenceid=16, filesize=11.7 K 2024-12-02T17:16:06,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 213eb7db536967e2cd87656b3cc2847c in 531ms, sequenceid=16, compaction requested=false 2024-12-02T17:16:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:06,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-02T17:16:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:16:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:06,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:07,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120256fb216b1fba40aaacc0575968d0d612_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159766400/Put/seqid=0 2024-12-02T17:16:07,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742229_1405 (size=12154) 2024-12-02T17:16:07,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:07,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159827147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159827159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159827162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159827165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159827166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159827278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159827297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T17:16:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159827307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159827308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159827307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:07,498 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120256fb216b1fba40aaacc0575968d0d612_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120256fb216b1fba40aaacc0575968d0d612_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:07,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/1c8645b0e35c4a0fbab8bbecfa127938, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:07,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/1c8645b0e35c4a0fbab8bbecfa127938 is 175, key is test_row_0/A:col10/1733159766400/Put/seqid=0 2024-12-02T17:16:07,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159827507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159827519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742230_1406 (size=30955) 2024-12-02T17:16:07,541 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/1c8645b0e35c4a0fbab8bbecfa127938 2024-12-02T17:16:07,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159827537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159827537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159827539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/37b7a01584fd42f9b7532cbf128fd300 is 50, key is test_row_0/B:col10/1733159766400/Put/seqid=0 2024-12-02T17:16:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742231_1407 (size=12001) 2024-12-02T17:16:07,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159827828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159827852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159827856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159827870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159827870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,013 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/37b7a01584fd42f9b7532cbf128fd300 2024-12-02T17:16:08,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/a5e5b96544cf4738bbd4c59778564191 is 50, key is test_row_0/C:col10/1733159766400/Put/seqid=0 2024-12-02T17:16:08,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742232_1408 (size=12001) 2024-12-02T17:16:08,088 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/a5e5b96544cf4738bbd4c59778564191 2024-12-02T17:16:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/1c8645b0e35c4a0fbab8bbecfa127938 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938 2024-12-02T17:16:08,127 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938, entries=150, sequenceid=36, filesize=30.2 K 2024-12-02T17:16:08,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/37b7a01584fd42f9b7532cbf128fd300 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/37b7a01584fd42f9b7532cbf128fd300 2024-12-02T17:16:08,141 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/37b7a01584fd42f9b7532cbf128fd300, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T17:16:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/a5e5b96544cf4738bbd4c59778564191 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a5e5b96544cf4738bbd4c59778564191 2024-12-02T17:16:08,155 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a5e5b96544cf4738bbd4c59778564191, entries=150, sequenceid=36, filesize=11.7 K 2024-12-02T17:16:08,161 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 213eb7db536967e2cd87656b3cc2847c in 1192ms, sequenceid=36, compaction requested=false 2024-12-02T17:16:08,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:08,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-02T17:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-02T17:16:08,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-02T17:16:08,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9940 sec 2024-12-02T17:16:08,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.9990 sec 2024-12-02T17:16:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T17:16:08,341 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:08,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-02T17:16:08,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:08,349 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:08,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:08,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:08,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:08,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c0104d99df9c4bb486d876c30fb975be_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:08,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742233_1409 (size=12154) 2024-12-02T17:16:08,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:08,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,509 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:16:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159828486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159828499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c0104d99df9c4bb486d876c30fb975be_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c0104d99df9c4bb486d876c30fb975be_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c7d59d86bcf64439a4e5cdb6b0046839, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:08,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c7d59d86bcf64439a4e5cdb6b0046839 is 175, key is test_row_0/A:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159828504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159828513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159828517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742234_1410 (size=30955) 2024-12-02T17:16:08,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159828617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159828631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159828637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159828637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159828640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:08,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:08,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159828837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159828857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159828857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159828857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159828867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:08,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:08,953 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c7d59d86bcf64439a4e5cdb6b0046839 2024-12-02T17:16:08,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:08,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/e51dbe304b16415f884596765a8b361b is 50, key is test_row_0/B:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:09,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742235_1411 (size=12001) 2024-12-02T17:16:09,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/e51dbe304b16415f884596765a8b361b 2024-12-02T17:16:09,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/55f1f1b22a3e40fc98954f0d291209f7 is 50, key is test_row_0/C:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:09,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742236_1412 (size=12001) 2024-12-02T17:16:09,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159829159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/55f1f1b22a3e40fc98954f0d291209f7 2024-12-02T17:16:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159829167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159829169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159829170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c7d59d86bcf64439a4e5cdb6b0046839 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839 2024-12-02T17:16:09,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159829205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839, entries=150, sequenceid=50, filesize=30.2 K 2024-12-02T17:16:09,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/e51dbe304b16415f884596765a8b361b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e51dbe304b16415f884596765a8b361b 2024-12-02T17:16:09,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:09,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e51dbe304b16415f884596765a8b361b, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T17:16:09,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/55f1f1b22a3e40fc98954f0d291209f7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/55f1f1b22a3e40fc98954f0d291209f7 2024-12-02T17:16:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:09,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/55f1f1b22a3e40fc98954f0d291209f7, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T17:16:09,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 213eb7db536967e2cd87656b3cc2847c in 948ms, sequenceid=50, compaction requested=true 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:09,326 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:09,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:09,326 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:09,337 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:09,337 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:09,337 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,338 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=90.7 K 2024-12-02T17:16:09,338 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,338 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839] 2024-12-02T17:16:09,338 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:09,339 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:09,339 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,339 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/8fb3276a11294eb7a376be509f475687, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/37b7a01584fd42f9b7532cbf128fd300, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e51dbe304b16415f884596765a8b361b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=35.2 K 2024-12-02T17:16:09,340 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64760b5fd36a49949ff233b38a1f33ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733159766199 2024-12-02T17:16:09,341 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8645b0e35c4a0fbab8bbecfa127938, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733159766398 2024-12-02T17:16:09,341 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb3276a11294eb7a376be509f475687, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733159766199 2024-12-02T17:16:09,342 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d59d86bcf64439a4e5cdb6b0046839, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159767158 2024-12-02T17:16:09,342 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b7a01584fd42f9b7532cbf128fd300, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733159766398 2024-12-02T17:16:09,351 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e51dbe304b16415f884596765a8b361b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159767158 2024-12-02T17:16:09,385 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:09,387 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:09,388 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/54bde81d79324bf18bd161b6ba44c92d is 50, key is test_row_0/B:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:09,398 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412020ed98f7de3264d4085ff259a7e24699b_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:09,401 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412020ed98f7de3264d4085ff259a7e24699b_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:09,401 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020ed98f7de3264d4085ff259a7e24699b_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:09,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742237_1413 (size=12104) 2024-12-02T17:16:09,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-02T17:16:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:09,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742238_1414 (size=4469) 2024-12-02T17:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:09,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026cda91f4379141f4a0f5337f697d84a7_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159768510/Put/seqid=0 2024-12-02T17:16:09,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742239_1415 (size=12154) 2024-12-02T17:16:09,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:09,607 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026cda91f4379141f4a0f5337f697d84a7_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026cda91f4379141f4a0f5337f697d84a7_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:09,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/047a450aa5da4677ac7ab28ae75056a8, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:09,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/047a450aa5da4677ac7ab28ae75056a8 is 175, key is test_row_0/A:col10/1733159768510/Put/seqid=0 2024-12-02T17:16:09,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742240_1416 (size=30955) 2024-12-02T17:16:09,685 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/047a450aa5da4677ac7ab28ae75056a8 2024-12-02T17:16:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:09,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/673fc76bf31e4e4893abe0aa86b0cb30 is 50, key is test_row_0/B:col10/1733159768510/Put/seqid=0 2024-12-02T17:16:09,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742241_1417 (size=12001) 2024-12-02T17:16:09,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159829756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159829757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159829767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159829780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159829785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,854 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/54bde81d79324bf18bd161b6ba44c92d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/54bde81d79324bf18bd161b6ba44c92d 2024-12-02T17:16:09,876 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into 54bde81d79324bf18bd161b6ba44c92d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:09,877 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:09,877 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=13, startTime=1733159769326; duration=0sec 2024-12-02T17:16:09,877 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:09,877 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:09,877 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:09,884 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:09,884 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:09,884 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:09,884 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ccc60f71413402aa4330567a6c7f6fe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a5e5b96544cf4738bbd4c59778564191, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/55f1f1b22a3e40fc98954f0d291209f7] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=35.2 K 2024-12-02T17:16:09,885 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ccc60f71413402aa4330567a6c7f6fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733159766199 2024-12-02T17:16:09,886 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e5b96544cf4738bbd4c59778564191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733159766398 2024-12-02T17:16:09,887 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f1f1b22a3e40fc98954f0d291209f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159767158 2024-12-02T17:16:09,887 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#357 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:09,887 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/8d3bfcaa788041748d11ed40b4cd4836 is 175, key is test_row_0/A:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159829898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159829898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159829898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159829898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,926 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:09,927 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/df17bec513f14b2ca7a4c8058b3073c5 is 50, key is test_row_0/C:col10/1733159768370/Put/seqid=0 2024-12-02T17:16:09,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742242_1418 (size=31058) 2024-12-02T17:16:09,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159829917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:09,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742243_1419 (size=12104) 2024-12-02T17:16:10,016 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/df17bec513f14b2ca7a4c8058b3073c5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/df17bec513f14b2ca7a4c8058b3073c5 2024-12-02T17:16:10,065 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into df17bec513f14b2ca7a4c8058b3073c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:10,065 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:10,065 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=13, startTime=1733159769326; duration=0sec 2024-12-02T17:16:10,065 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:10,065 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159830117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159830118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159830133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159830137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,166 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/673fc76bf31e4e4893abe0aa86b0cb30 2024-12-02T17:16:10,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159830162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/fbb19ffb8c1144b3be69a49751dbf292 is 50, key is test_row_0/C:col10/1733159768510/Put/seqid=0 2024-12-02T17:16:10,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742244_1420 (size=12001) 2024-12-02T17:16:10,364 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/8d3bfcaa788041748d11ed40b4cd4836 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8d3bfcaa788041748d11ed40b4cd4836 2024-12-02T17:16:10,384 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into 8d3bfcaa788041748d11ed40b4cd4836(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-02T17:16:10,384 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:10,384 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=13, startTime=1733159769326; duration=1sec 2024-12-02T17:16:10,384 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:10,384 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:10,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159830450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159830450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159830452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159830453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159830488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:10,622 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/fbb19ffb8c1144b3be69a49751dbf292 2024-12-02T17:16:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/047a450aa5da4677ac7ab28ae75056a8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8 2024-12-02T17:16:10,668 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8, entries=150, sequenceid=68, filesize=30.2 K 2024-12-02T17:16:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/673fc76bf31e4e4893abe0aa86b0cb30 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/673fc76bf31e4e4893abe0aa86b0cb30 2024-12-02T17:16:10,683 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/673fc76bf31e4e4893abe0aa86b0cb30, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T17:16:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/fbb19ffb8c1144b3be69a49751dbf292 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/fbb19ffb8c1144b3be69a49751dbf292 2024-12-02T17:16:10,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/fbb19ffb8c1144b3be69a49751dbf292, entries=150, sequenceid=68, filesize=11.7 K 2024-12-02T17:16:10,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 213eb7db536967e2cd87656b3cc2847c in 1240ms, sequenceid=68, compaction requested=false 2024-12-02T17:16:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:10,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-02T17:16:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-02T17:16:10,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-02T17:16:10,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-12-02T17:16:10,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3930 sec 2024-12-02T17:16:10,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:16:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:10,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:10,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:10,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:10,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029b45617cb5784851ba994c2a2383751d_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742245_1421 (size=14594) 2024-12-02T17:16:11,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:11,021 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029b45617cb5784851ba994c2a2383751d_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029b45617cb5784851ba994c2a2383751d_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:11,022 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f84ce5068cf5450c86cf753bfeb92fc6, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:11,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f84ce5068cf5450c86cf753bfeb92fc6 is 175, key is test_row_0/A:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742246_1422 (size=39549) 2024-12-02T17:16:11,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f84ce5068cf5450c86cf753bfeb92fc6 2024-12-02T17:16:11,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159831086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159831098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/fba51cab641a46f4b040dc0d8b696c9d is 50, key is test_row_0/B:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:11,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159831106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159831111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159831115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742247_1423 (size=12001) 2024-12-02T17:16:11,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/fba51cab641a46f4b040dc0d8b696c9d 2024-12-02T17:16:11,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/a228ea552cde44e88b4de76cf5e7d24b is 50, key is test_row_0/C:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:11,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159831227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159831227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159831240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159831247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159831256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742248_1424 (size=12001) 2024-12-02T17:16:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159831461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159831464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159831469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159831469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159831488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/a228ea552cde44e88b4de76cf5e7d24b 2024-12-02T17:16:11,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f84ce5068cf5450c86cf753bfeb92fc6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6 2024-12-02T17:16:11,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159831778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6, entries=200, sequenceid=86, filesize=38.6 K 2024-12-02T17:16:11,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/fba51cab641a46f4b040dc0d8b696c9d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/fba51cab641a46f4b040dc0d8b696c9d 2024-12-02T17:16:11,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159831797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159831797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159831801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/fba51cab641a46f4b040dc0d8b696c9d, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T17:16:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/a228ea552cde44e88b4de76cf5e7d24b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a228ea552cde44e88b4de76cf5e7d24b 2024-12-02T17:16:11,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159831821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:11,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a228ea552cde44e88b4de76cf5e7d24b, entries=150, sequenceid=86, filesize=11.7 K 2024-12-02T17:16:11,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 213eb7db536967e2cd87656b3cc2847c in 909ms, sequenceid=86, compaction requested=true 2024-12-02T17:16:11,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:11,885 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:11,886 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:11,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:11,892 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:11,893 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:11,893 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:11,893 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8d3bfcaa788041748d11ed40b4cd4836, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=99.2 K 2024-12-02T17:16:11,893 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:11,893 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8d3bfcaa788041748d11ed40b4cd4836, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6] 2024-12-02T17:16:11,900 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d3bfcaa788041748d11ed40b4cd4836, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159767158 2024-12-02T17:16:11,901 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:11,901 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:11,901 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:11,902 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/54bde81d79324bf18bd161b6ba44c92d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/673fc76bf31e4e4893abe0aa86b0cb30, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/fba51cab641a46f4b040dc0d8b696c9d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=35.3 K 2024-12-02T17:16:11,902 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047a450aa5da4677ac7ab28ae75056a8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733159768462 2024-12-02T17:16:11,903 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 54bde81d79324bf18bd161b6ba44c92d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159767158 2024-12-02T17:16:11,904 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84ce5068cf5450c86cf753bfeb92fc6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159769730 2024-12-02T17:16:11,907 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 673fc76bf31e4e4893abe0aa86b0cb30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733159768462 2024-12-02T17:16:11,915 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fba51cab641a46f4b040dc0d8b696c9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159769730 2024-12-02T17:16:11,993 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:11,997 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:11,997 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/e4ac61be34ee458297a080462185c964 is 50, key is test_row_0/B:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:11,999 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202d7da830e53cf452bac8042a7b46a31bf_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:12,001 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202d7da830e53cf452bac8042a7b46a31bf_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:12,001 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d7da830e53cf452bac8042a7b46a31bf_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742249_1425 (size=12207) 2024-12-02T17:16:12,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742250_1426 (size=4469) 2024-12-02T17:16:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:12,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:12,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:12,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:12,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:12,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:12,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:12,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:12,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bccbe37bcb0749ceaae4382b85621b4f_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159771098/Put/seqid=0 2024-12-02T17:16:12,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742251_1427 (size=14594) 2024-12-02T17:16:12,442 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/e4ac61be34ee458297a080462185c964 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e4ac61be34ee458297a080462185c964 2024-12-02T17:16:12,452 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into e4ac61be34ee458297a080462185c964(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:12,452 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:12,452 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=13, startTime=1733159771886; duration=0sec 2024-12-02T17:16:12,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159832389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,452 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:12,453 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:12,453 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159832393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,456 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#366 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159832389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,456 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/003560d767c34fd1a46c3d756ed3af71 is 175, key is test_row_0/A:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,457 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:12,457 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:12,457 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:12,458 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/df17bec513f14b2ca7a4c8058b3073c5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/fbb19ffb8c1144b3be69a49751dbf292, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a228ea552cde44e88b4de76cf5e7d24b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=35.3 K 2024-12-02T17:16:12,458 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting df17bec513f14b2ca7a4c8058b3073c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159767158 2024-12-02T17:16:12,458 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb19ffb8c1144b3be69a49751dbf292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733159768462 2024-12-02T17:16:12,459 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a228ea552cde44e88b4de76cf5e7d24b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159769730 2024-12-02T17:16:12,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742252_1428 (size=31161) 2024-12-02T17:16:12,481 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/003560d767c34fd1a46c3d756ed3af71 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/003560d767c34fd1a46c3d756ed3af71 2024-12-02T17:16:12,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159832453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,488 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into 003560d767c34fd1a46c3d756ed3af71(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:12,488 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:12,488 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=13, startTime=1733159771885; duration=0sec 2024-12-02T17:16:12,488 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:12,488 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:12,493 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:12,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159832459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,494 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/aa7fa5cc33cf431bbdd16c0748062609 is 50, key is test_row_0/C:col10/1733159769730/Put/seqid=0 2024-12-02T17:16:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T17:16:12,507 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:12,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-02T17:16:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:12,509 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:12,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:12,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:12,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742253_1429 (size=12207) 2024-12-02T17:16:12,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159832559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159832560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159832568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159832597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159832597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:12,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:12,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:12,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:12,766 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bccbe37bcb0749ceaae4382b85621b4f_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bccbe37bcb0749ceaae4382b85621b4f_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:12,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/7403d1fd08a847c09d00e4a5fd7324b0, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:12,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/7403d1fd08a847c09d00e4a5fd7324b0 is 175, key is test_row_0/A:col10/1733159771098/Put/seqid=0 2024-12-02T17:16:12,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159832779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159832787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742254_1430 (size=39549) 2024-12-02T17:16:12,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159832799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:12,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:12,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:12,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159832828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159832829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:12,953 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/aa7fa5cc33cf431bbdd16c0748062609 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/aa7fa5cc33cf431bbdd16c0748062609 2024-12-02T17:16:12,959 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into aa7fa5cc33cf431bbdd16c0748062609(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:12,959 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:12,959 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=13, startTime=1733159771886; duration=0sec 2024-12-02T17:16:12,959 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:12,959 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:13,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159833107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159833107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159833119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159833147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159833148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:13,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:13,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,213 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/7403d1fd08a847c09d00e4a5fd7324b0 2024-12-02T17:16:13,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/462c9866e16048e18ccdf507d3fa4c10 is 50, key is test_row_0/B:col10/1733159771098/Put/seqid=0 2024-12-02T17:16:13,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742255_1431 (size=12001) 2024-12-02T17:16:13,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/462c9866e16048e18ccdf507d3fa4c10 2024-12-02T17:16:13,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/938c0755f8584596bf33605552ceede0 is 50, key is test_row_0/C:col10/1733159771098/Put/seqid=0 2024-12-02T17:16:13,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742256_1432 (size=12001) 2024-12-02T17:16:13,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/938c0755f8584596bf33605552ceede0 2024-12-02T17:16:13,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/7403d1fd08a847c09d00e4a5fd7324b0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0 2024-12-02T17:16:13,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0, entries=200, sequenceid=104, filesize=38.6 K 2024-12-02T17:16:13,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/462c9866e16048e18ccdf507d3fa4c10 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/462c9866e16048e18ccdf507d3fa4c10 2024-12-02T17:16:13,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/462c9866e16048e18ccdf507d3fa4c10, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T17:16:13,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/938c0755f8584596bf33605552ceede0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/938c0755f8584596bf33605552ceede0 2024-12-02T17:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:13,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/938c0755f8584596bf33605552ceede0, entries=150, sequenceid=104, filesize=11.7 K 2024-12-02T17:16:13,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 213eb7db536967e2cd87656b3cc2847c in 1186ms, sequenceid=104, compaction requested=false 2024-12-02T17:16:13,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:13,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-02T17:16:13,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:13,632 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:13,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:13,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c48cea5819c2415b97ee7b724d6ed843_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159772415/Put/seqid=0 2024-12-02T17:16:13,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742257_1433 (size=12154) 2024-12-02T17:16:13,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:13,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c48cea5819c2415b97ee7b724d6ed843_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c48cea5819c2415b97ee7b724d6ed843_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/a228944c05de47318679a2630d818403, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:13,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/a228944c05de47318679a2630d818403 is 175, key is test_row_0/A:col10/1733159772415/Put/seqid=0 2024-12-02T17:16:13,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159833703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159833709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159833716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742258_1434 (size=30955) 2024-12-02T17:16:13,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159833743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159833744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159833849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159833851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159833864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159833887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159833892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159834069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159834081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159834088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159834109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159834121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,167 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/a228944c05de47318679a2630d818403 2024-12-02T17:16:14,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/df17883a16dd45a7a3a014f6e7fd48c1 is 50, key is test_row_0/B:col10/1733159772415/Put/seqid=0 2024-12-02T17:16:14,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742259_1435 (size=12001) 2024-12-02T17:16:14,240 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/df17883a16dd45a7a3a014f6e7fd48c1 2024-12-02T17:16:14,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/ffc5105eb92c46488697f7af4e70db5e is 50, key is test_row_0/C:col10/1733159772415/Put/seqid=0 2024-12-02T17:16:14,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742260_1436 (size=12001) 2024-12-02T17:16:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159834389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159834407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159834417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159834442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159834449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:14,794 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/ffc5105eb92c46488697f7af4e70db5e 2024-12-02T17:16:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/a228944c05de47318679a2630d818403 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403 2024-12-02T17:16:14,803 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403, entries=150, sequenceid=121, filesize=30.2 K 2024-12-02T17:16:14,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/df17883a16dd45a7a3a014f6e7fd48c1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/df17883a16dd45a7a3a014f6e7fd48c1 2024-12-02T17:16:14,819 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/df17883a16dd45a7a3a014f6e7fd48c1, entries=150, sequenceid=121, filesize=11.7 K 2024-12-02T17:16:14,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/ffc5105eb92c46488697f7af4e70db5e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/ffc5105eb92c46488697f7af4e70db5e 2024-12-02T17:16:14,835 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/ffc5105eb92c46488697f7af4e70db5e, entries=150, sequenceid=121, filesize=11.7 K 2024-12-02T17:16:14,841 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 213eb7db536967e2cd87656b3cc2847c in 1209ms, sequenceid=121, compaction requested=true 2024-12-02T17:16:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:14,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-02T17:16:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-02T17:16:14,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-02T17:16:14,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3330 sec 2024-12-02T17:16:14,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3420 sec 2024-12-02T17:16:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:14,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:16:14,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:14,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:14,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:14,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:14,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:14,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:14,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023e8a1eefcacf4aea96209c5e9e878d32_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742261_1437 (size=14794) 2024-12-02T17:16:15,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159835001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159835014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159835017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159835021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,053 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412023e8a1eefcacf4aea96209c5e9e878d32_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023e8a1eefcacf4aea96209c5e9e878d32_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:15,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159835029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c081d10d38ef4efb8f6bc905d2a80898, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:15,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c081d10d38ef4efb8f6bc905d2a80898 is 175, key is test_row_0/A:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:15,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742262_1438 (size=39749) 2024-12-02T17:16:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159835138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159835141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159835152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159835157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159835173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159835367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159835367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159835388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159835392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159835393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,518 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c081d10d38ef4efb8f6bc905d2a80898 2024-12-02T17:16:15,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/2cf84c3023cf412495fdfb30ca7d3a36 is 50, key is test_row_0/B:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:15,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742263_1439 (size=12151) 2024-12-02T17:16:15,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/2cf84c3023cf412495fdfb30ca7d3a36 2024-12-02T17:16:15,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/816fc486131b420ebb31b8e071cb0649 is 50, key is test_row_0/C:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:15,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742264_1440 (size=12151) 2024-12-02T17:16:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159835677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159835689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159835708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159835709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159835717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:15,753 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-02T17:16:15,753 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-02T17:16:16,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/816fc486131b420ebb31b8e071cb0649 2024-12-02T17:16:16,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/c081d10d38ef4efb8f6bc905d2a80898 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898 2024-12-02T17:16:16,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898, entries=200, sequenceid=140, filesize=38.8 K 2024-12-02T17:16:16,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/2cf84c3023cf412495fdfb30ca7d3a36 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/2cf84c3023cf412495fdfb30ca7d3a36 2024-12-02T17:16:16,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/2cf84c3023cf412495fdfb30ca7d3a36, entries=150, sequenceid=140, filesize=11.9 K 2024-12-02T17:16:16,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/816fc486131b420ebb31b8e071cb0649 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/816fc486131b420ebb31b8e071cb0649 2024-12-02T17:16:16,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/816fc486131b420ebb31b8e071cb0649, entries=150, sequenceid=140, filesize=11.9 K 2024-12-02T17:16:16,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 213eb7db536967e2cd87656b3cc2847c in 1171ms, sequenceid=140, compaction requested=true 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:16,110 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:16,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:16,110 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:16,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:16,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:16,114 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:16,114 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e4ac61be34ee458297a080462185c964, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/462c9866e16048e18ccdf507d3fa4c10, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/df17883a16dd45a7a3a014f6e7fd48c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/2cf84c3023cf412495fdfb30ca7d3a36] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=47.2 K 2024-12-02T17:16:16,115 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ac61be34ee458297a080462185c964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159769730 2024-12-02T17:16:16,115 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:16,115 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:16,115 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:16,115 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/003560d767c34fd1a46c3d756ed3af71, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=138.1 K 2024-12-02T17:16:16,115 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:16,115 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/003560d767c34fd1a46c3d756ed3af71, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898] 2024-12-02T17:16:16,116 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462c9866e16048e18ccdf507d3fa4c10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733159771098 2024-12-02T17:16:16,116 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 003560d767c34fd1a46c3d756ed3af71, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159769730 2024-12-02T17:16:16,117 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting df17883a16dd45a7a3a014f6e7fd48c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733159772383 2024-12-02T17:16:16,117 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 7403d1fd08a847c09d00e4a5fd7324b0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733159771098 2024-12-02T17:16:16,117 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cf84c3023cf412495fdfb30ca7d3a36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159773696 2024-12-02T17:16:16,118 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a228944c05de47318679a2630d818403, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733159772383 2024-12-02T17:16:16,122 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c081d10d38ef4efb8f6bc905d2a80898, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159773696 2024-12-02T17:16:16,157 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:16,186 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#379 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:16,186 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/444a601baef449b08bb662c4afd1cc2a is 50, key is test_row_0/B:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:16,191 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412027f80b08e59174c089a474a9ebf5d5fe3_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:16,193 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412027f80b08e59174c089a474a9ebf5d5fe3_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:16,194 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412027f80b08e59174c089a474a9ebf5d5fe3_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:16,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742265_1441 (size=12493) 2024-12-02T17:16:16,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:16:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:16,228 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/444a601baef449b08bb662c4afd1cc2a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/444a601baef449b08bb662c4afd1cc2a 2024-12-02T17:16:16,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742266_1442 (size=4469) 2024-12-02T17:16:16,249 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#378 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:16,249 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/3a73e67b30e541cd8d5b7ad904c3a766 is 175, key is test_row_0/A:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:16,251 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into 444a601baef449b08bb662c4afd1cc2a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:16,251 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:16,251 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=12, startTime=1733159776110; duration=0sec 2024-12-02T17:16:16,251 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:16,251 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:16,251 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:16,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f4acdd63099847f5a075ed42d3cd4d35_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159774993/Put/seqid=0 2024-12-02T17:16:16,255 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:16,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:16,256 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:16,256 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/aa7fa5cc33cf431bbdd16c0748062609, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/938c0755f8584596bf33605552ceede0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/ffc5105eb92c46488697f7af4e70db5e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/816fc486131b420ebb31b8e071cb0649] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=47.2 K 2024-12-02T17:16:16,257 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7fa5cc33cf431bbdd16c0748062609, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733159769730 2024-12-02T17:16:16,257 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938c0755f8584596bf33605552ceede0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733159771098 2024-12-02T17:16:16,257 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc5105eb92c46488697f7af4e70db5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733159772383 2024-12-02T17:16:16,259 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 816fc486131b420ebb31b8e071cb0649, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159773696 2024-12-02T17:16:16,284 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:16,286 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e is 50, key is test_row_0/C:col10/1733159774936/Put/seqid=0 2024-12-02T17:16:16,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742268_1444 (size=14794) 2024-12-02T17:16:16,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742267_1443 (size=31447) 2024-12-02T17:16:16,335 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/3a73e67b30e541cd8d5b7ad904c3a766 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/3a73e67b30e541cd8d5b7ad904c3a766 2024-12-02T17:16:16,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742269_1445 (size=12493) 2024-12-02T17:16:16,353 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into 3a73e67b30e541cd8d5b7ad904c3a766(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:16,353 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:16,353 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=12, startTime=1733159776110; duration=0sec 2024-12-02T17:16:16,353 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:16,353 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:16,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159836335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159836346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159836360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,388 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e 2024-12-02T17:16:16,410 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into 6ba9a4fdb71a4c71bf2ac85a5bcdbb4e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:16,410 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:16,411 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=12, startTime=1733159776110; duration=0sec 2024-12-02T17:16:16,411 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:16,411 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159836377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159836382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159836492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159836494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159836496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159836517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159836532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T17:16:16,692 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:16,693 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-02T17:16:16,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T17:16:16,696 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:16,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:16,699 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159836726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f4acdd63099847f5a075ed42d3cd4d35_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f4acdd63099847f5a075ed42d3cd4d35_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159836726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159836727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159836737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,766 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f0f953fd455a4e49951e98748695589e, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:16,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f0f953fd455a4e49951e98748695589e is 175, key is test_row_0/A:col10/1733159774993/Put/seqid=0 2024-12-02T17:16:16,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159836758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:16,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T17:16:16,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742270_1446 (size=39749) 2024-12-02T17:16:16,844 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f0f953fd455a4e49951e98748695589e 2024-12-02T17:16:16,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T17:16:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:16,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:16,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/cd8b13412a5a44898eb5ca3181c9825c is 50, key is test_row_0/B:col10/1733159774993/Put/seqid=0 2024-12-02T17:16:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742271_1447 (size=12151) 2024-12-02T17:16:17,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T17:16:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:17,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T17:16:17,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159837067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159837068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159837068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159837068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159837109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T17:16:17,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T17:16:17,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/cd8b13412a5a44898eb5ca3181c9825c 2024-12-02T17:16:17,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T17:16:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:17,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:17,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/79a39778e08e4fd18a26542e2d971b8e is 50, key is test_row_0/C:col10/1733159774993/Put/seqid=0 2024-12-02T17:16:17,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742272_1448 (size=12151) 2024-12-02T17:16:17,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/79a39778e08e4fd18a26542e2d971b8e 2024-12-02T17:16:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/f0f953fd455a4e49951e98748695589e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e 2024-12-02T17:16:17,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e, entries=200, sequenceid=155, filesize=38.8 K 2024-12-02T17:16:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/cd8b13412a5a44898eb5ca3181c9825c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/cd8b13412a5a44898eb5ca3181c9825c 2024-12-02T17:16:17,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/cd8b13412a5a44898eb5ca3181c9825c, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T17:16:17,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/79a39778e08e4fd18a26542e2d971b8e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/79a39778e08e4fd18a26542e2d971b8e 2024-12-02T17:16:17,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/79a39778e08e4fd18a26542e2d971b8e, entries=150, sequenceid=155, filesize=11.9 K 2024-12-02T17:16:17,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 213eb7db536967e2cd87656b3cc2847c in 1254ms, sequenceid=155, compaction requested=false 2024-12-02T17:16:17,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:17,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-02T17:16:17,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:17,496 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:17,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b8b530629a1846be946975fac2fa9d1d_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159776273/Put/seqid=0 2024-12-02T17:16:17,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742273_1449 (size=12304) 2024-12-02T17:16:17,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:17,543 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b8b530629a1846be946975fac2fa9d1d_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b8b530629a1846be946975fac2fa9d1d_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:17,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b40c422f7389428fb1024bd2dd3649dc, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:17,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b40c422f7389428fb1024bd2dd3649dc is 175, key is test_row_0/A:col10/1733159776273/Put/seqid=0 2024-12-02T17:16:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742274_1450 (size=31105) 2024-12-02T17:16:17,599 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b40c422f7389428fb1024bd2dd3649dc 2024-12-02T17:16:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:17,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:17,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/4ece4aeb91b24a44a4bacdc7c216d698 is 50, key is test_row_0/B:col10/1733159776273/Put/seqid=0 2024-12-02T17:16:17,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742275_1451 (size=12151) 2024-12-02T17:16:17,664 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/4ece4aeb91b24a44a4bacdc7c216d698 2024-12-02T17:16:17,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/3180e148b3214e498c102582ce0bffe8 is 50, key is test_row_0/C:col10/1733159776273/Put/seqid=0 2024-12-02T17:16:17,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742276_1452 (size=12151) 2024-12-02T17:16:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159837677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159837707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159837783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159837785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159837785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T17:16:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159837888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159837891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159837927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159837927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:17,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159837927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159838116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159838122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,145 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/3180e148b3214e498c102582ce0bffe8 2024-12-02T17:16:18,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b40c422f7389428fb1024bd2dd3649dc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc 2024-12-02T17:16:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159838157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159838157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159838158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,176 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc, entries=150, sequenceid=176, filesize=30.4 K 2024-12-02T17:16:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/4ece4aeb91b24a44a4bacdc7c216d698 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4ece4aeb91b24a44a4bacdc7c216d698 2024-12-02T17:16:18,191 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4ece4aeb91b24a44a4bacdc7c216d698, entries=150, sequenceid=176, filesize=11.9 K 2024-12-02T17:16:18,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/3180e148b3214e498c102582ce0bffe8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3180e148b3214e498c102582ce0bffe8 2024-12-02T17:16:18,212 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3180e148b3214e498c102582ce0bffe8, entries=150, sequenceid=176, filesize=11.9 K 2024-12-02T17:16:18,214 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 213eb7db536967e2cd87656b3cc2847c in 717ms, sequenceid=176, compaction requested=true 2024-12-02T17:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-02T17:16:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-02T17:16:18,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-02T17:16:18,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5220 sec 2024-12-02T17:16:18,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5340 sec 2024-12-02T17:16:18,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T17:16:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:18,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a86c4c2dd4f6454f93cba145b2852610_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:18,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742277_1453 (size=17284) 2024-12-02T17:16:18,525 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:18,540 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202a86c4c2dd4f6454f93cba145b2852610_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a86c4c2dd4f6454f93cba145b2852610_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:18,543 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/e2132d22b8eb493eb78169a025c79bb2, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:18,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/e2132d22b8eb493eb78169a025c79bb2 is 175, key is test_row_0/A:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:18,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742278_1454 (size=48389) 2024-12-02T17:16:18,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159838550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159838561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159838562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159838562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159838564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159838689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159838709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159838709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159838711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159838711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-02T17:16:18,867 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:18,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-02T17:16:18,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:18,871 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:18,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:18,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/e2132d22b8eb493eb78169a025c79bb2 2024-12-02T17:16:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:18,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159838933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159838938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159838947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159838949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159838949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/38a1262a94d04789b2bb62fdb790f937 is 50, key is test_row_0/B:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:19,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T17:16:19,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:19,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742279_1455 (size=12151) 2024-12-02T17:16:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:19,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T17:16:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159839290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159839290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159839302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159839305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159839306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T17:16:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/38a1262a94d04789b2bb62fdb790f937 2024-12-02T17:16:19,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/56ee95fca2b64889bedf8bb55e5b8060 is 50, key is test_row_0/C:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:19,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742280_1456 (size=12151) 2024-12-02T17:16:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:19,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/56ee95fca2b64889bedf8bb55e5b8060 2024-12-02T17:16:19,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T17:16:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:19,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/e2132d22b8eb493eb78169a025c79bb2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2 2024-12-02T17:16:19,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2, entries=250, sequenceid=194, filesize=47.3 K 2024-12-02T17:16:19,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/38a1262a94d04789b2bb62fdb790f937 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/38a1262a94d04789b2bb62fdb790f937 2024-12-02T17:16:19,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/38a1262a94d04789b2bb62fdb790f937, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T17:16:19,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/56ee95fca2b64889bedf8bb55e5b8060 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/56ee95fca2b64889bedf8bb55e5b8060 2024-12-02T17:16:19,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/56ee95fca2b64889bedf8bb55e5b8060, entries=150, sequenceid=194, filesize=11.9 K 2024-12-02T17:16:19,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 213eb7db536967e2cd87656b3cc2847c in 1070ms, sequenceid=194, compaction requested=true 2024-12-02T17:16:19,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:19,544 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:19,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:19,545 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:19,548 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150690 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:19,548 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:19,548 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,549 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/3a73e67b30e541cd8d5b7ad904c3a766, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=147.2 K 2024-12-02T17:16:19,549 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,549 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/3a73e67b30e541cd8d5b7ad904c3a766, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2] 2024-12-02T17:16:19,549 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:19,549 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:19,549 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,549 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/444a601baef449b08bb662c4afd1cc2a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/cd8b13412a5a44898eb5ca3181c9825c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4ece4aeb91b24a44a4bacdc7c216d698, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/38a1262a94d04789b2bb62fdb790f937] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=47.8 K 2024-12-02T17:16:19,551 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a73e67b30e541cd8d5b7ad904c3a766, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159773696 2024-12-02T17:16:19,551 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 444a601baef449b08bb662c4afd1cc2a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159773696 2024-12-02T17:16:19,553 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f953fd455a4e49951e98748695589e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733159774993 2024-12-02T17:16:19,554 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40c422f7389428fb1024bd2dd3649dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733159776273 2024-12-02T17:16:19,554 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2132d22b8eb493eb78169a025c79bb2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733159777635 2024-12-02T17:16:19,557 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting cd8b13412a5a44898eb5ca3181c9825c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733159774993 2024-12-02T17:16:19,559 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ece4aeb91b24a44a4bacdc7c216d698, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733159776273 2024-12-02T17:16:19,564 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 38a1262a94d04789b2bb62fdb790f937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733159777729 2024-12-02T17:16:19,573 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:19,595 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:19,596 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/98e5ec3d51314382bc88d65dccf9cd29 is 50, key is test_row_0/B:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:19,597 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120254af05e657764622be9c6f277bf5ec5d_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:19,600 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120254af05e657764622be9c6f277bf5ec5d_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:19,600 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120254af05e657764622be9c6f277bf5ec5d_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:19,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742281_1457 (size=12629) 2024-12-02T17:16:19,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-02T17:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,663 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:19,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:19,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:19,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:19,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:19,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:19,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742282_1458 (size=4469) 2024-12-02T17:16:19,675 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#390 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:19,675 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/039d283198dd48cc980f66a25d71a68a is 175, key is test_row_0/A:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:19,680 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/98e5ec3d51314382bc88d65dccf9cd29 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/98e5ec3d51314382bc88d65dccf9cd29 2024-12-02T17:16:19,706 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into 98e5ec3d51314382bc88d65dccf9cd29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:19,706 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:19,707 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=12, startTime=1733159779545; duration=0sec 2024-12-02T17:16:19,707 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:19,707 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:19,707 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:19,709 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:19,709 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:19,709 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:19,709 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/79a39778e08e4fd18a26542e2d971b8e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3180e148b3214e498c102582ce0bffe8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/56ee95fca2b64889bedf8bb55e5b8060] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=47.8 K 2024-12-02T17:16:19,710 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba9a4fdb71a4c71bf2ac85a5bcdbb4e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159773696 2024-12-02T17:16:19,710 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a39778e08e4fd18a26542e2d971b8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733159774993 2024-12-02T17:16:19,710 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 3180e148b3214e498c102582ce0bffe8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733159776273 2024-12-02T17:16:19,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202feef894b261f4f6aa7e4f0f9e7f5b945_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159778554/Put/seqid=0 2024-12-02T17:16:19,722 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ee95fca2b64889bedf8bb55e5b8060, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733159777729 2024-12-02T17:16:19,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742283_1459 (size=31583) 2024-12-02T17:16:19,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742284_1460 (size=12304) 2024-12-02T17:16:19,775 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:19,776 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/3d0efa520a764088b902b25bfc15934f is 50, key is test_row_0/C:col10/1733159777729/Put/seqid=0 2024-12-02T17:16:19,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742285_1461 (size=12629) 2024-12-02T17:16:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:19,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:19,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159839908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159839909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159839910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159839924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:19,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159839929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159840037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159840037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159840037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159840037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159840058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,132 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/039d283198dd48cc980f66a25d71a68a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/039d283198dd48cc980f66a25d71a68a 2024-12-02T17:16:20,138 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into 039d283198dd48cc980f66a25d71a68a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:20,138 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:20,138 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=12, startTime=1733159779544; duration=0sec 2024-12-02T17:16:20,138 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:20,138 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:20,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:20,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202feef894b261f4f6aa7e4f0f9e7f5b945_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202feef894b261f4f6aa7e4f0f9e7f5b945_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:20,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/5d4ae61dd6134319a27753948294d969, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:20,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/5d4ae61dd6134319a27753948294d969 is 175, key is test_row_0/A:col10/1733159778554/Put/seqid=0 2024-12-02T17:16:20,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742286_1462 (size=31105) 2024-12-02T17:16:20,248 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/3d0efa520a764088b902b25bfc15934f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3d0efa520a764088b902b25bfc15934f 2024-12-02T17:16:20,270 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into 3d0efa520a764088b902b25bfc15934f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:20,270 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:20,270 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=12, startTime=1733159779545; duration=0sec 2024-12-02T17:16:20,270 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:20,270 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:20,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159840257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159840258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159840258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159840259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159840280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159840587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159840587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159840587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159840587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159840607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:20,642 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/5d4ae61dd6134319a27753948294d969 2024-12-02T17:16:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/167407c4eb7c448abcccebf59434d195 is 50, key is test_row_0/B:col10/1733159778554/Put/seqid=0 2024-12-02T17:16:20,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742287_1463 (size=12151) 2024-12-02T17:16:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159841096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:21,106 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/167407c4eb7c448abcccebf59434d195 2024-12-02T17:16:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:21,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159841097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:21,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159841107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159841107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:21,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4f2ee474c06a4cdd9ab7c3e35bbc445c is 50, key is test_row_0/C:col10/1733159778554/Put/seqid=0 2024-12-02T17:16:21,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159841129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:21,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742288_1464 (size=12151) 2024-12-02T17:16:21,158 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4f2ee474c06a4cdd9ab7c3e35bbc445c 2024-12-02T17:16:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/5d4ae61dd6134319a27753948294d969 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969 2024-12-02T17:16:21,166 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969, entries=150, sequenceid=208, filesize=30.4 K 2024-12-02T17:16:21,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/167407c4eb7c448abcccebf59434d195 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/167407c4eb7c448abcccebf59434d195 2024-12-02T17:16:21,173 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/167407c4eb7c448abcccebf59434d195, entries=150, sequenceid=208, filesize=11.9 K 2024-12-02T17:16:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4f2ee474c06a4cdd9ab7c3e35bbc445c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f2ee474c06a4cdd9ab7c3e35bbc445c 2024-12-02T17:16:21,178 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f2ee474c06a4cdd9ab7c3e35bbc445c, entries=150, sequenceid=208, filesize=11.9 K 2024-12-02T17:16:21,180 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 213eb7db536967e2cd87656b3cc2847c in 1517ms, sequenceid=208, compaction requested=false 2024-12-02T17:16:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-02T17:16:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-02T17:16:21,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-02T17:16:21,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3110 sec 2024-12-02T17:16:21,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3180 sec 2024-12-02T17:16:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:22,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:16:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:22,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cb7048bb27ff4c0d86405f33ba21a77e_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:22,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742289_1465 (size=12304) 2024-12-02T17:16:22,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159842139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159842147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,154 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159842151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159842153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159842153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159842256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159842267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159842267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159842267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159842477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159842487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159842487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159842487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,536 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:22,541 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cb7048bb27ff4c0d86405f33ba21a77e_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cb7048bb27ff4c0d86405f33ba21a77e_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:22,542 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/8b8f3f715a0f4496a24ca8af6aeb326e, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:22,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/8b8f3f715a0f4496a24ca8af6aeb326e is 175, key is test_row_0/A:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:22,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742290_1466 (size=31105) 2024-12-02T17:16:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159842786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159842797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159842797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159842797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:22,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/8b8f3f715a0f4496a24ca8af6aeb326e 2024-12-02T17:16:22,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/72b95e8bac5e48e9852b6d0a05157742 is 50, key is test_row_0/B:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:22,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742291_1467 (size=12151) 2024-12-02T17:16:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-02T17:16:23,027 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:23,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-02T17:16:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T17:16:23,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:23,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:23,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T17:16:23,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T17:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:23,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:23,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159843297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159843316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159843317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159843317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:23,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T17:16:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T17:16:23,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/72b95e8bac5e48e9852b6d0a05157742 2024-12-02T17:16:23,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/9df35d50a06644989c677f864870fdbe is 50, key is test_row_0/C:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:23,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742292_1468 (size=12151) 2024-12-02T17:16:23,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/9df35d50a06644989c677f864870fdbe 2024-12-02T17:16:23,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/8b8f3f715a0f4496a24ca8af6aeb326e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e 2024-12-02T17:16:23,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e, entries=150, sequenceid=231, filesize=30.4 K 2024-12-02T17:16:23,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/72b95e8bac5e48e9852b6d0a05157742 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/72b95e8bac5e48e9852b6d0a05157742 2024-12-02T17:16:23,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/72b95e8bac5e48e9852b6d0a05157742, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T17:16:23,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/9df35d50a06644989c677f864870fdbe as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9df35d50a06644989c677f864870fdbe 2024-12-02T17:16:23,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9df35d50a06644989c677f864870fdbe, entries=150, sequenceid=231, filesize=11.9 K 2024-12-02T17:16:23,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 213eb7db536967e2cd87656b3cc2847c in 1285ms, sequenceid=231, compaction requested=true 2024-12-02T17:16:23,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:23,410 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:23,410 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:23,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:23,411 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:23,411 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:23,411 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,411 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/039d283198dd48cc980f66a25d71a68a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=91.6 K 2024-12-02T17:16:23,411 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,411 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/039d283198dd48cc980f66a25d71a68a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e] 2024-12-02T17:16:23,411 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:23,411 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:23,411 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039d283198dd48cc980f66a25d71a68a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733159777729 2024-12-02T17:16:23,411 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,411 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/98e5ec3d51314382bc88d65dccf9cd29, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/167407c4eb7c448abcccebf59434d195, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/72b95e8bac5e48e9852b6d0a05157742] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=36.1 K 2024-12-02T17:16:23,412 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d4ae61dd6134319a27753948294d969, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733159778554 2024-12-02T17:16:23,413 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e5ec3d51314382bc88d65dccf9cd29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733159777729 2024-12-02T17:16:23,415 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b8f3f715a0f4496a24ca8af6aeb326e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159779922 2024-12-02T17:16:23,416 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 167407c4eb7c448abcccebf59434d195, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733159778554 2024-12-02T17:16:23,416 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b95e8bac5e48e9852b6d0a05157742, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159779922 2024-12-02T17:16:23,424 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:23,427 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120283814a841fd04cb8ab30b40182d290cf_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:23,429 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120283814a841fd04cb8ab30b40182d290cf_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:23,429 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120283814a841fd04cb8ab30b40182d290cf_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:23,444 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:23,445 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/ad76491e1cb7474388ce098b3461b139 is 50, key is test_row_0/B:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:23,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742293_1469 (size=4469) 2024-12-02T17:16:23,466 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#399 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:23,467 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/a4a1bc1d11d64b69a7d16df8020e23db is 175, key is test_row_0/A:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:23,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742294_1470 (size=12731) 2024-12-02T17:16:23,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742295_1471 (size=31685) 2024-12-02T17:16:23,482 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/ad76491e1cb7474388ce098b3461b139 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ad76491e1cb7474388ce098b3461b139 2024-12-02T17:16:23,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-02T17:16:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,489 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:16:23,491 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into ad76491e1cb7474388ce098b3461b139(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:23,491 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:23,492 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=13, startTime=1733159783410; duration=0sec 2024-12-02T17:16:23,492 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:23,492 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:23,492 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:23,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:23,493 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:23,493 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:23,493 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:23,494 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3d0efa520a764088b902b25bfc15934f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f2ee474c06a4cdd9ab7c3e35bbc445c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9df35d50a06644989c677f864870fdbe] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=36.1 K 2024-12-02T17:16:23,495 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d0efa520a764088b902b25bfc15934f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733159777729 2024-12-02T17:16:23,495 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2ee474c06a4cdd9ab7c3e35bbc445c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733159778554 2024-12-02T17:16:23,496 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9df35d50a06644989c677f864870fdbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159779922 2024-12-02T17:16:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eba0b34178be46c6b52f39653240a2f4_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159782151/Put/seqid=0 2024-12-02T17:16:23,510 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:23,511 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4e105c4f26594164adaa0486c28225ed is 50, key is test_row_0/C:col10/1733159779922/Put/seqid=0 2024-12-02T17:16:23,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742296_1472 (size=12304) 2024-12-02T17:16:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:23,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742297_1473 (size=12731) 2024-12-02T17:16:23,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202eba0b34178be46c6b52f39653240a2f4_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eba0b34178be46c6b52f39653240a2f4_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:23,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/55c0b06d90b54810a5a50fe73e6626c1, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/55c0b06d90b54810a5a50fe73e6626c1 is 175, key is test_row_0/A:col10/1733159782151/Put/seqid=0 2024-12-02T17:16:23,526 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4e105c4f26594164adaa0486c28225ed as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4e105c4f26594164adaa0486c28225ed 2024-12-02T17:16:23,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742298_1474 (size=31105) 2024-12-02T17:16:23,533 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/55c0b06d90b54810a5a50fe73e6626c1 2024-12-02T17:16:23,533 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into 4e105c4f26594164adaa0486c28225ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:23,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:23,533 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=13, startTime=1733159783410; duration=0sec 2024-12-02T17:16:23,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:23,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:23,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/1c6b9783e2194dce8a7b19756bbce7e6 is 50, key is test_row_0/B:col10/1733159782151/Put/seqid=0 2024-12-02T17:16:23,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742299_1475 (size=12151) 2024-12-02T17:16:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T17:16:23,884 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/a4a1bc1d11d64b69a7d16df8020e23db as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a4a1bc1d11d64b69a7d16df8020e23db 2024-12-02T17:16:23,890 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into a4a1bc1d11d64b69a7d16df8020e23db(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:23,890 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:23,890 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=13, startTime=1733159783410; duration=0sec 2024-12-02T17:16:23,890 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:23,890 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:23,950 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/1c6b9783e2194dce8a7b19756bbce7e6 2024-12-02T17:16:23,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4059049a7de34c79bd91936c4acaaf0f is 50, key is test_row_0/C:col10/1733159782151/Put/seqid=0 2024-12-02T17:16:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742300_1476 (size=12151) 2024-12-02T17:16:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T17:16:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:24,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:24,245 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T17:16:24,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159844323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159844324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159844337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159844337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159844337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,345 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,345 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,346 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,384 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4059049a7de34c79bd91936c4acaaf0f 2024-12-02T17:16:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/55c0b06d90b54810a5a50fe73e6626c1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1 2024-12-02T17:16:24,393 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1, entries=150, sequenceid=244, filesize=30.4 K 2024-12-02T17:16:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/1c6b9783e2194dce8a7b19756bbce7e6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1c6b9783e2194dce8a7b19756bbce7e6 2024-12-02T17:16:24,397 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1c6b9783e2194dce8a7b19756bbce7e6, entries=150, sequenceid=244, filesize=11.9 K 2024-12-02T17:16:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4059049a7de34c79bd91936c4acaaf0f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4059049a7de34c79bd91936c4acaaf0f 2024-12-02T17:16:24,401 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4059049a7de34c79bd91936c4acaaf0f, entries=150, sequenceid=244, filesize=11.9 K 2024-12-02T17:16:24,402 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 213eb7db536967e2cd87656b3cc2847c in 913ms, sequenceid=244, compaction requested=false 2024-12-02T17:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-02T17:16:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-02T17:16:24,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-02T17:16:24,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3740 sec 2024-12-02T17:16:24,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.3780 sec 2024-12-02T17:16:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:24,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T17:16:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:24,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:24,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029d18cd6d75164a3db063fedeadaea742_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742301_1477 (size=14994) 2024-12-02T17:16:24,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159844480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159844482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159844587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159844596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159844807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159844817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:24,858 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:24,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029d18cd6d75164a3db063fedeadaea742_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029d18cd6d75164a3db063fedeadaea742_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:24,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b6a02e17100a426b95df2e1a439267cb, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:24,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b6a02e17100a426b95df2e1a439267cb is 175, key is test_row_0/A:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:24,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742302_1478 (size=39949) 2024-12-02T17:16:24,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b6a02e17100a426b95df2e1a439267cb 2024-12-02T17:16:24,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb is 50, key is test_row_0/B:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:24,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742303_1479 (size=12301) 2024-12-02T17:16:24,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb 2024-12-02T17:16:24,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/0648c64051734add89d5b8b0e373a4be is 50, key is test_row_0/C:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:24,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742304_1480 (size=12301) 2024-12-02T17:16:24,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/0648c64051734add89d5b8b0e373a4be 2024-12-02T17:16:24,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b6a02e17100a426b95df2e1a439267cb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb 2024-12-02T17:16:24,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb, entries=200, sequenceid=267, filesize=39.0 K 2024-12-02T17:16:24,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb 2024-12-02T17:16:24,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb, entries=150, sequenceid=267, filesize=12.0 K 2024-12-02T17:16:24,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/0648c64051734add89d5b8b0e373a4be as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0648c64051734add89d5b8b0e373a4be 2024-12-02T17:16:24,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0648c64051734add89d5b8b0e373a4be, entries=150, sequenceid=267, filesize=12.0 K 2024-12-02T17:16:24,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 213eb7db536967e2cd87656b3cc2847c in 479ms, sequenceid=267, compaction requested=true 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:24,920 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:24,920 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:24,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:24,923 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:24,923 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:24,923 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:24,923 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ad76491e1cb7474388ce098b3461b139, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1c6b9783e2194dce8a7b19756bbce7e6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=36.3 K 2024-12-02T17:16:24,923 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:24,924 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:24,924 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:24,924 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a4a1bc1d11d64b69a7d16df8020e23db, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=100.3 K 2024-12-02T17:16:24,924 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:24,924 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a4a1bc1d11d64b69a7d16df8020e23db, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb] 2024-12-02T17:16:24,926 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a1bc1d11d64b69a7d16df8020e23db, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159779922 2024-12-02T17:16:24,926 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ad76491e1cb7474388ce098b3461b139, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159779922 2024-12-02T17:16:24,926 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c6b9783e2194dce8a7b19756bbce7e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733159782125 2024-12-02T17:16:24,926 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c0b06d90b54810a5a50fe73e6626c1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733159782125 2024-12-02T17:16:24,926 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1ede9c7d1f4e4c843735d2bd8dbfbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733159784307 2024-12-02T17:16:24,927 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6a02e17100a426b95df2e1a439267cb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733159784307 2024-12-02T17:16:24,934 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:24,935 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:24,935 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/463cef4af5d941d0ba9082430a2e0eb9 is 50, key is test_row_0/B:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:24,945 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202915f7546a0d543cf8feed090b55f04a9_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:24,947 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202915f7546a0d543cf8feed090b55f04a9_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:24,947 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202915f7546a0d543cf8feed090b55f04a9_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:24,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742305_1481 (size=12983) 2024-12-02T17:16:24,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742306_1482 (size=4469) 2024-12-02T17:16:24,977 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#409 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:24,978 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/493e7d21b10a4c3b9e0150782ab54490 is 175, key is test_row_0/A:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:24,978 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/463cef4af5d941d0ba9082430a2e0eb9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/463cef4af5d941d0ba9082430a2e0eb9 2024-12-02T17:16:24,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742307_1483 (size=31937) 2024-12-02T17:16:24,993 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into 463cef4af5d941d0ba9082430a2e0eb9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:24,993 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:24,993 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=13, startTime=1733159784920; duration=0sec 2024-12-02T17:16:24,993 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:24,993 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:24,993 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:24,994 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:24,994 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:24,994 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:24,994 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4e105c4f26594164adaa0486c28225ed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4059049a7de34c79bd91936c4acaaf0f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0648c64051734add89d5b8b0e373a4be] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=36.3 K 2024-12-02T17:16:24,994 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e105c4f26594164adaa0486c28225ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733159779922 2024-12-02T17:16:24,995 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4059049a7de34c79bd91936c4acaaf0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733159782125 2024-12-02T17:16:24,995 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0648c64051734add89d5b8b0e373a4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733159784307 2024-12-02T17:16:25,007 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:25,007 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/1ba57345120845388100bb4c9ec7ba24 is 50, key is test_row_0/C:col10/1733159784439/Put/seqid=0 2024-12-02T17:16:25,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742308_1484 (size=12983) 2024-12-02T17:16:25,040 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/1ba57345120845388100bb4c9ec7ba24 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ba57345120845388100bb4c9ec7ba24 2024-12-02T17:16:25,048 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into 1ba57345120845388100bb4c9ec7ba24(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:25,048 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:25,048 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=13, startTime=1733159784920; duration=0sec 2024-12-02T17:16:25,049 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:25,049 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:25,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:16:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:25,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025912ca62928e4308becbb98b43de6f77_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159784470/Put/seqid=0 2024-12-02T17:16:25,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742309_1485 (size=14994) 2024-12-02T17:16:25,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,154 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412025912ca62928e4308becbb98b43de6f77_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025912ca62928e4308becbb98b43de6f77_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:25,155 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d02c0b83aa72473884505849d31e3c9b, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:25,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d02c0b83aa72473884505849d31e3c9b is 175, key is test_row_0/A:col10/1733159784470/Put/seqid=0 2024-12-02T17:16:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742310_1486 (size=39949) 2024-12-02T17:16:25,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d02c0b83aa72473884505849d31e3c9b 2024-12-02T17:16:25,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/6af3cecbffae462bba0a1bab2be00023 is 50, key is test_row_0/B:col10/1733159784470/Put/seqid=0 2024-12-02T17:16:25,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T17:16:25,177 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:25,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-02T17:16:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:25,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:25,181 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:25,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:25,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742311_1487 (size=12301) 2024-12-02T17:16:25,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/6af3cecbffae462bba0a1bab2be00023 2024-12-02T17:16:25,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/0c2d780871ad461191fda0e0fb9ea7ee is 50, key is test_row_0/C:col10/1733159784470/Put/seqid=0 2024-12-02T17:16:25,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742312_1488 (size=12301) 2024-12-02T17:16:25,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/0c2d780871ad461191fda0e0fb9ea7ee 2024-12-02T17:16:25,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d02c0b83aa72473884505849d31e3c9b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b 2024-12-02T17:16:25,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b, entries=200, sequenceid=281, filesize=39.0 K 2024-12-02T17:16:25,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/6af3cecbffae462bba0a1bab2be00023 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/6af3cecbffae462bba0a1bab2be00023 2024-12-02T17:16:25,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/6af3cecbffae462bba0a1bab2be00023, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T17:16:25,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/0c2d780871ad461191fda0e0fb9ea7ee as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0c2d780871ad461191fda0e0fb9ea7ee 2024-12-02T17:16:25,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0c2d780871ad461191fda0e0fb9ea7ee, entries=150, sequenceid=281, filesize=12.0 K 2024-12-02T17:16:25,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=107.34 KB/109920 for 213eb7db536967e2cd87656b3cc2847c in 115ms, sequenceid=281, compaction requested=false 2024-12-02T17:16:25,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:25,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:25,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120221e30c860bc84fd3afa44c34d77f8f23_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159785234/Put/seqid=0 2024-12-02T17:16:25,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742313_1489 (size=14994) 2024-12-02T17:16:25,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120221e30c860bc84fd3afa44c34d77f8f23_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120221e30c860bc84fd3afa44c34d77f8f23_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:25,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/863c98349a9447549051fa9612c34d0d, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:25,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/863c98349a9447549051fa9612c34d0d is 175, key is test_row_0/A:col10/1733159785234/Put/seqid=0 2024-12-02T17:16:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:25,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159845290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159845297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742314_1490 (size=39949) 2024-12-02T17:16:25,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/863c98349a9447549051fa9612c34d0d 2024-12-02T17:16:25,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/ae02e8011efe4a9290dcb8a5e158c3e0 is 50, key is test_row_0/B:col10/1733159785234/Put/seqid=0 2024-12-02T17:16:25,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T17:16:25,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742315_1491 (size=12301) 2024-12-02T17:16:25,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/ae02e8011efe4a9290dcb8a5e158c3e0 2024-12-02T17:16:25,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/9af46e551a60476794015ca84daaf3e5 is 50, key is test_row_0/C:col10/1733159785234/Put/seqid=0 2024-12-02T17:16:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742316_1492 (size=12301) 2024-12-02T17:16:25,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/9af46e551a60476794015ca84daaf3e5 2024-12-02T17:16:25,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/863c98349a9447549051fa9612c34d0d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d 2024-12-02T17:16:25,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d, entries=200, sequenceid=301, filesize=39.0 K 2024-12-02T17:16:25,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/ae02e8011efe4a9290dcb8a5e158c3e0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ae02e8011efe4a9290dcb8a5e158c3e0 2024-12-02T17:16:25,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ae02e8011efe4a9290dcb8a5e158c3e0, entries=150, sequenceid=301, filesize=12.0 K 2024-12-02T17:16:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/9af46e551a60476794015ca84daaf3e5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9af46e551a60476794015ca84daaf3e5 2024-12-02T17:16:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9af46e551a60476794015ca84daaf3e5, entries=150, sequenceid=301, filesize=12.0 K 2024-12-02T17:16:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 213eb7db536967e2cd87656b3cc2847c in 147ms, sequenceid=301, compaction requested=true 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,390 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:25,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:25,391 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/493e7d21b10a4c3b9e0150782ab54490 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/493e7d21b10a4c3b9e0150782ab54490 2024-12-02T17:16:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,391 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:16:25,391 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:16:25,391 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. because compaction request was cancelled 2024-12-02T17:16:25,391 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:25,391 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,392 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:25,392 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:25,392 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,392 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/463cef4af5d941d0ba9082430a2e0eb9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/6af3cecbffae462bba0a1bab2be00023, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ae02e8011efe4a9290dcb8a5e158c3e0] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=36.7 K 2024-12-02T17:16:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,392 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 463cef4af5d941d0ba9082430a2e0eb9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733159784307 2024-12-02T17:16:25,393 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6af3cecbffae462bba0a1bab2be00023, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733159784469 2024-12-02T17:16:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,394 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ae02e8011efe4a9290dcb8a5e158c3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733159785218 2024-12-02T17:16:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,396 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into 493e7d21b10a4c3b9e0150782ab54490(size=31.2 K), total size for store is 109.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:25,396 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:25,396 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=13, startTime=1733159784920; duration=0sec 2024-12-02T17:16:25,396 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:25,396 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:25,396 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,397 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:25,397 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,397 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,398 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ba57345120845388100bb4c9ec7ba24, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0c2d780871ad461191fda0e0fb9ea7ee, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9af46e551a60476794015ca84daaf3e5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=36.7 K 2024-12-02T17:16:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,398 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ba57345120845388100bb4c9ec7ba24, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733159784307 2024-12-02T17:16:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,399 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2d780871ad461191fda0e0fb9ea7ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733159784469 2024-12-02T17:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,400 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af46e551a60476794015ca84daaf3e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733159785218 2024-12-02T17:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,402 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:25,402 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/95b0498023ac4c00abab9b02b371395c is 50, key is test_row_0/B:col10/1733159785234/Put/seqid=0 2024-12-02T17:16:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742317_1493 (size=13085) 2024-12-02T17:16:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,424 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/95b0498023ac4c00abab9b02b371395c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/95b0498023ac4c00abab9b02b371395c 2024-12-02T17:16:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,430 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into 95b0498023ac4c00abab9b02b371395c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:25,430 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:25,430 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=13, startTime=1733159785390; duration=0sec 2024-12-02T17:16:25,430 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:25,430 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,435 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,435 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/33ba8bcb95e240ecac485a715da4e44f is 50, key is test_row_0/C:col10/1733159785234/Put/seqid=0 2024-12-02T17:16:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:25,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:16:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:25,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742318_1494 (size=13085) 2024-12-02T17:16:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T17:16:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:25,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bb385d6ca62a4a0e9703e367b1411a8d_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159785446/Put/seqid=0 2024-12-02T17:16:25,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742319_1495 (size=12454) 2024-12-02T17:16:25,507 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:25,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202bb385d6ca62a4a0e9703e367b1411a8d_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bb385d6ca62a4a0e9703e367b1411a8d_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:25,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d2fedd6ede27456daffa62a2b1f319da, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:25,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d2fedd6ede27456daffa62a2b1f319da is 175, key is test_row_0/A:col10/1733159785446/Put/seqid=0 2024-12-02T17:16:25,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742320_1496 (size=31255) 2024-12-02T17:16:25,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159845556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159845570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T17:16:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159845677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159845677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T17:16:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:25,863 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/33ba8bcb95e240ecac485a715da4e44f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/33ba8bcb95e240ecac485a715da4e44f 2024-12-02T17:16:25,866 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into 33ba8bcb95e240ecac485a715da4e44f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:25,866 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:25,866 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=13, startTime=1733159785390; duration=0sec 2024-12-02T17:16:25,867 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:25,867 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56054 deadline: 1733159845887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159845887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:25,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d2fedd6ede27456daffa62a2b1f319da 2024-12-02T17:16:25,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/4c347c36749c45f3a1e2b3ee13a7edf9 is 50, key is test_row_0/B:col10/1733159785446/Put/seqid=0 2024-12-02T17:16:25,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T17:16:25,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:25,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742321_1497 (size=12301) 2024-12-02T17:16:25,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:25,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/4c347c36749c45f3a1e2b3ee13a7edf9 2024-12-02T17:16:25,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/c35119b862fa4deb995b07d691125cf8 is 50, key is test_row_0/C:col10/1733159785446/Put/seqid=0 2024-12-02T17:16:25,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742322_1498 (size=12301) 2024-12-02T17:16:25,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/c35119b862fa4deb995b07d691125cf8 2024-12-02T17:16:25,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/d2fedd6ede27456daffa62a2b1f319da as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da 2024-12-02T17:16:25,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da, entries=150, sequenceid=316, filesize=30.5 K 2024-12-02T17:16:25,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/4c347c36749c45f3a1e2b3ee13a7edf9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4c347c36749c45f3a1e2b3ee13a7edf9 2024-12-02T17:16:25,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4c347c36749c45f3a1e2b3ee13a7edf9, entries=150, sequenceid=316, filesize=12.0 K 2024-12-02T17:16:25,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/c35119b862fa4deb995b07d691125cf8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/c35119b862fa4deb995b07d691125cf8 2024-12-02T17:16:26,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/c35119b862fa4deb995b07d691125cf8, entries=150, sequenceid=316, filesize=12.0 K 2024-12-02T17:16:26,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 213eb7db536967e2cd87656b3cc2847c in 557ms, sequenceid=316, compaction requested=true 2024-12-02T17:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:26,006 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:26,006 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:26,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:26,007 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. because compaction request was cancelled 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:26,008 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:26,008 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/493e7d21b10a4c3b9e0150782ab54490, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=139.7 K 2024-12-02T17:16:26,008 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:26,008 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/493e7d21b10a4c3b9e0150782ab54490, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da] 2024-12-02T17:16:26,010 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 493e7d21b10a4c3b9e0150782ab54490, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733159784307 2024-12-02T17:16:26,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:16:26,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:16:26,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. because compaction request was cancelled 2024-12-02T17:16:26,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:26,011 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d02c0b83aa72473884505849d31e3c9b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733159784469 2024-12-02T17:16:26,011 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863c98349a9447549051fa9612c34d0d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733159785218 2024-12-02T17:16:26,011 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2fedd6ede27456daffa62a2b1f319da, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733159785288 2024-12-02T17:16:26,019 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:26,029 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202aff1a8c8294c4d148bddf774c21136da_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:26,032 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202aff1a8c8294c4d148bddf774c21136da_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:26,032 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202aff1a8c8294c4d148bddf774c21136da_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742323_1499 (size=4469) 2024-12-02T17:16:26,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-02T17:16:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:26,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:16:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:26,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:26,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:26,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020dc48d0bc65e426c9a76528aacf5a1a9_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159785555/Put/seqid=0 2024-12-02T17:16:26,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742324_1500 (size=12454) 2024-12-02T17:16:26,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:26,132 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020dc48d0bc65e426c9a76528aacf5a1a9_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020dc48d0bc65e426c9a76528aacf5a1a9_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/4dc8befdbe1748ffae75c2ff7eadeecf, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/4dc8befdbe1748ffae75c2ff7eadeecf is 175, key is test_row_0/A:col10/1733159785555/Put/seqid=0 2024-12-02T17:16:26,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742325_1501 (size=31255) 2024-12-02T17:16:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:26,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. as already flushing 2024-12-02T17:16:26,253 INFO [Thread-1805 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1805. 2024-12-02T17:16:26,253 DEBUG [Thread-1805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:26,253 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,253 DEBUG [Thread-1805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:26,254 INFO [Thread-1790 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1790. 2024-12-02T17:16:26,254 DEBUG [Thread-1790 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:26,255 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,255 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,259 INFO [Thread-1807 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1807. 2024-12-02T17:16:26,259 DEBUG [Thread-1807 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:26,259 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,259 DEBUG [Thread-1807 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:26,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:26,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159846243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,262 INFO [Thread-1799 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1799. 2024-12-02T17:16:26,262 DEBUG [Thread-1799 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:26,262 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:26,263 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,264 INFO [Thread-1803 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1803. 2024-12-02T17:16:26,264 DEBUG [Thread-1803 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:26,264 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,264 DEBUG [Thread-1803 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:26,270 INFO [Thread-1801 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1801. 2024-12-02T17:16:26,270 DEBUG [Thread-1801 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:26,270 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,270 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:26,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:26,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56040 deadline: 1733159846356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,357 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:26,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159846367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:55996 deadline: 1733159846367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56066 deadline: 1733159846367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,367 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,367 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 213eb7db536967e2cd87656b3cc2847c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,441 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#422 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-12-02T17:16:26,441 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/afb8db509d59475987fd628bf555abf5 is 175, key is test_row_0/A:col10/1733159785446/Put/seqid=0 2024-12-02T17:16:26,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742326_1502 (size=32073) 2024-12-02T17:16:26,537 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/4dc8befdbe1748ffae75c2ff7eadeecf 2024-12-02T17:16:26,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/45ea45edf6b54282aee8400c04e3f80e is 50, key is test_row_0/B:col10/1733159785555/Put/seqid=0 2024-12-02T17:16:26,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742327_1503 (size=12301) 2024-12-02T17:16:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159846576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,848 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/afb8db509d59475987fd628bf555abf5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/afb8db509d59475987fd628bf555abf5 2024-12-02T17:16:26,852 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into afb8db509d59475987fd628bf555abf5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:26,852 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:26,852 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=12, startTime=1733159786006; duration=0sec 2024-12-02T17:16:26,852 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:26,852 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:26,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56016 deadline: 1733159846886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:16:26,947 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/45ea45edf6b54282aee8400c04e3f80e 2024-12-02T17:16:26,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/6c66f11d845f453a84f28ba99c46e829 is 50, key is test_row_0/C:col10/1733159785555/Put/seqid=0 2024-12-02T17:16:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742328_1504 (size=12301) 2024-12-02T17:16:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:27,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/6c66f11d845f453a84f28ba99c46e829 2024-12-02T17:16:27,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/4dc8befdbe1748ffae75c2ff7eadeecf as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf 2024-12-02T17:16:27,363 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf, entries=150, sequenceid=336, filesize=30.5 K 2024-12-02T17:16:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/45ea45edf6b54282aee8400c04e3f80e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/45ea45edf6b54282aee8400c04e3f80e 2024-12-02T17:16:27,366 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/45ea45edf6b54282aee8400c04e3f80e, entries=150, sequenceid=336, filesize=12.0 K 2024-12-02T17:16:27,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/6c66f11d845f453a84f28ba99c46e829 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6c66f11d845f453a84f28ba99c46e829 2024-12-02T17:16:27,369 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6c66f11d845f453a84f28ba99c46e829, entries=150, sequenceid=336, filesize=12.0 K 2024-12-02T17:16:27,370 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 213eb7db536967e2cd87656b3cc2847c in 1267ms, sequenceid=336, compaction requested=true 2024-12-02T17:16:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:27,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-02T17:16:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-02T17:16:27,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-02T17:16:27,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1900 sec 2024-12-02T17:16:27,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.1950 sec 2024-12-02T17:16:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:27,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:27,398 INFO [Thread-1796 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1796. 2024-12-02T17:16:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:27,398 DEBUG [Thread-1796 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:27,398 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:27,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:27,398 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:27,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:27,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fe2ecaeddb40452c869a7fcd59ded125_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_0/A:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:27,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742329_1505 (size=12454) 2024-12-02T17:16:27,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:27,810 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202fe2ecaeddb40452c869a7fcd59ded125_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fe2ecaeddb40452c869a7fcd59ded125_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:27,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/56a6361677eb4e7886bde420727151ae, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:27,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/56a6361677eb4e7886bde420727151ae is 175, key is test_row_0/A:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:27,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742330_1506 (size=31255) 2024-12-02T17:16:28,214 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/56a6361677eb4e7886bde420727151ae 2024-12-02T17:16:28,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/a9ac37ee0030468f80ef3f9ac1e791ca is 50, key is test_row_0/B:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742331_1507 (size=12301) 2024-12-02T17:16:28,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/a9ac37ee0030468f80ef3f9ac1e791ca 2024-12-02T17:16:28,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/8fa77f16cd214218971deb306e688714 is 50, key is test_row_0/C:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:28,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742332_1508 (size=12301) 2024-12-02T17:16:28,812 DEBUG [regionserver/3b3375d1a925:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-12-02T17:16:28,846 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8d3bfcaa788041748d11ed40b4cd4836, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/003560d767c34fd1a46c3d756ed3af71, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/3a73e67b30e541cd8d5b7ad904c3a766, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/039d283198dd48cc980f66a25d71a68a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a4a1bc1d11d64b69a7d16df8020e23db, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/493e7d21b10a4c3b9e0150782ab54490, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da] to archive 2024-12-02T17:16:28,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:28,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/64760b5fd36a49949ff233b38a1f33ea 2024-12-02T17:16:28,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/1c8645b0e35c4a0fbab8bbecfa127938 2024-12-02T17:16:28,850 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8d3bfcaa788041748d11ed40b4cd4836 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8d3bfcaa788041748d11ed40b4cd4836 2024-12-02T17:16:28,851 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c7d59d86bcf64439a4e5cdb6b0046839 2024-12-02T17:16:28,852 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/047a450aa5da4677ac7ab28ae75056a8 2024-12-02T17:16:28,853 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f84ce5068cf5450c86cf753bfeb92fc6 2024-12-02T17:16:28,854 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/003560d767c34fd1a46c3d756ed3af71 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/003560d767c34fd1a46c3d756ed3af71 2024-12-02T17:16:28,855 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/7403d1fd08a847c09d00e4a5fd7324b0 2024-12-02T17:16:28,856 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a228944c05de47318679a2630d818403 2024-12-02T17:16:28,857 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/c081d10d38ef4efb8f6bc905d2a80898 2024-12-02T17:16:28,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/3a73e67b30e541cd8d5b7ad904c3a766 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/3a73e67b30e541cd8d5b7ad904c3a766 2024-12-02T17:16:28,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/f0f953fd455a4e49951e98748695589e 2024-12-02T17:16:28,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b40c422f7389428fb1024bd2dd3649dc 2024-12-02T17:16:28,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/e2132d22b8eb493eb78169a025c79bb2 2024-12-02T17:16:28,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/039d283198dd48cc980f66a25d71a68a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/039d283198dd48cc980f66a25d71a68a 2024-12-02T17:16:28,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/5d4ae61dd6134319a27753948294d969 2024-12-02T17:16:28,864 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a4a1bc1d11d64b69a7d16df8020e23db to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/a4a1bc1d11d64b69a7d16df8020e23db 2024-12-02T17:16:28,864 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/8b8f3f715a0f4496a24ca8af6aeb326e 2024-12-02T17:16:28,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/55c0b06d90b54810a5a50fe73e6626c1 2024-12-02T17:16:28,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b6a02e17100a426b95df2e1a439267cb 2024-12-02T17:16:28,867 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/493e7d21b10a4c3b9e0150782ab54490 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/493e7d21b10a4c3b9e0150782ab54490 2024-12-02T17:16:28,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d02c0b83aa72473884505849d31e3c9b 2024-12-02T17:16:28,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/863c98349a9447549051fa9612c34d0d 2024-12-02T17:16:28,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/d2fedd6ede27456daffa62a2b1f319da 2024-12-02T17:16:28,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/8fb3276a11294eb7a376be509f475687, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/37b7a01584fd42f9b7532cbf128fd300, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/54bde81d79324bf18bd161b6ba44c92d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e51dbe304b16415f884596765a8b361b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/673fc76bf31e4e4893abe0aa86b0cb30, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e4ac61be34ee458297a080462185c964, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/fba51cab641a46f4b040dc0d8b696c9d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/462c9866e16048e18ccdf507d3fa4c10, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/df17883a16dd45a7a3a014f6e7fd48c1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/444a601baef449b08bb662c4afd1cc2a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/2cf84c3023cf412495fdfb30ca7d3a36, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/cd8b13412a5a44898eb5ca3181c9825c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4ece4aeb91b24a44a4bacdc7c216d698, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/98e5ec3d51314382bc88d65dccf9cd29, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/38a1262a94d04789b2bb62fdb790f937, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/167407c4eb7c448abcccebf59434d195, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ad76491e1cb7474388ce098b3461b139, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/72b95e8bac5e48e9852b6d0a05157742, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1c6b9783e2194dce8a7b19756bbce7e6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/463cef4af5d941d0ba9082430a2e0eb9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/6af3cecbffae462bba0a1bab2be00023, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ae02e8011efe4a9290dcb8a5e158c3e0] to archive 2024-12-02T17:16:28,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:28,874 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/8fb3276a11294eb7a376be509f475687 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/8fb3276a11294eb7a376be509f475687 2024-12-02T17:16:28,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/37b7a01584fd42f9b7532cbf128fd300 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/37b7a01584fd42f9b7532cbf128fd300 2024-12-02T17:16:28,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/54bde81d79324bf18bd161b6ba44c92d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/54bde81d79324bf18bd161b6ba44c92d 2024-12-02T17:16:28,876 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e51dbe304b16415f884596765a8b361b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e51dbe304b16415f884596765a8b361b 2024-12-02T17:16:28,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/673fc76bf31e4e4893abe0aa86b0cb30 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/673fc76bf31e4e4893abe0aa86b0cb30 2024-12-02T17:16:28,878 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e4ac61be34ee458297a080462185c964 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/e4ac61be34ee458297a080462185c964 2024-12-02T17:16:28,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/fba51cab641a46f4b040dc0d8b696c9d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/fba51cab641a46f4b040dc0d8b696c9d 2024-12-02T17:16:28,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/462c9866e16048e18ccdf507d3fa4c10 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/462c9866e16048e18ccdf507d3fa4c10 2024-12-02T17:16:28,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/df17883a16dd45a7a3a014f6e7fd48c1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/df17883a16dd45a7a3a014f6e7fd48c1 2024-12-02T17:16:28,881 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/444a601baef449b08bb662c4afd1cc2a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/444a601baef449b08bb662c4afd1cc2a 2024-12-02T17:16:28,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/2cf84c3023cf412495fdfb30ca7d3a36 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/2cf84c3023cf412495fdfb30ca7d3a36 2024-12-02T17:16:28,883 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/cd8b13412a5a44898eb5ca3181c9825c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/cd8b13412a5a44898eb5ca3181c9825c 2024-12-02T17:16:28,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4ece4aeb91b24a44a4bacdc7c216d698 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4ece4aeb91b24a44a4bacdc7c216d698 2024-12-02T17:16:28,885 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/98e5ec3d51314382bc88d65dccf9cd29 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/98e5ec3d51314382bc88d65dccf9cd29 2024-12-02T17:16:28,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/38a1262a94d04789b2bb62fdb790f937 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/38a1262a94d04789b2bb62fdb790f937 2024-12-02T17:16:28,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/167407c4eb7c448abcccebf59434d195 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/167407c4eb7c448abcccebf59434d195 2024-12-02T17:16:28,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ad76491e1cb7474388ce098b3461b139 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ad76491e1cb7474388ce098b3461b139 2024-12-02T17:16:28,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/72b95e8bac5e48e9852b6d0a05157742 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/72b95e8bac5e48e9852b6d0a05157742 2024-12-02T17:16:28,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1c6b9783e2194dce8a7b19756bbce7e6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1c6b9783e2194dce8a7b19756bbce7e6 2024-12-02T17:16:28,891 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/463cef4af5d941d0ba9082430a2e0eb9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/463cef4af5d941d0ba9082430a2e0eb9 2024-12-02T17:16:28,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/1f1ede9c7d1f4e4c843735d2bd8dbfbb 2024-12-02T17:16:28,893 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/6af3cecbffae462bba0a1bab2be00023 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/6af3cecbffae462bba0a1bab2be00023 2024-12-02T17:16:28,894 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ae02e8011efe4a9290dcb8a5e158c3e0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/ae02e8011efe4a9290dcb8a5e158c3e0 2024-12-02T17:16:28,898 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ccc60f71413402aa4330567a6c7f6fe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a5e5b96544cf4738bbd4c59778564191, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/df17bec513f14b2ca7a4c8058b3073c5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/55f1f1b22a3e40fc98954f0d291209f7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/fbb19ffb8c1144b3be69a49751dbf292, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/aa7fa5cc33cf431bbdd16c0748062609, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a228ea552cde44e88b4de76cf5e7d24b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/938c0755f8584596bf33605552ceede0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/ffc5105eb92c46488697f7af4e70db5e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/816fc486131b420ebb31b8e071cb0649, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/79a39778e08e4fd18a26542e2d971b8e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3180e148b3214e498c102582ce0bffe8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3d0efa520a764088b902b25bfc15934f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/56ee95fca2b64889bedf8bb55e5b8060, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f2ee474c06a4cdd9ab7c3e35bbc445c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4e105c4f26594164adaa0486c28225ed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9df35d50a06644989c677f864870fdbe, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4059049a7de34c79bd91936c4acaaf0f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ba57345120845388100bb4c9ec7ba24, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0648c64051734add89d5b8b0e373a4be, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0c2d780871ad461191fda0e0fb9ea7ee, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9af46e551a60476794015ca84daaf3e5] to archive 2024-12-02T17:16:28,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:28,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ccc60f71413402aa4330567a6c7f6fe to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ccc60f71413402aa4330567a6c7f6fe 2024-12-02T17:16:28,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a5e5b96544cf4738bbd4c59778564191 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a5e5b96544cf4738bbd4c59778564191 2024-12-02T17:16:28,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/df17bec513f14b2ca7a4c8058b3073c5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/df17bec513f14b2ca7a4c8058b3073c5 2024-12-02T17:16:28,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/55f1f1b22a3e40fc98954f0d291209f7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/55f1f1b22a3e40fc98954f0d291209f7 2024-12-02T17:16:28,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/fbb19ffb8c1144b3be69a49751dbf292 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/fbb19ffb8c1144b3be69a49751dbf292 2024-12-02T17:16:28,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/aa7fa5cc33cf431bbdd16c0748062609 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/aa7fa5cc33cf431bbdd16c0748062609 2024-12-02T17:16:28,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a228ea552cde44e88b4de76cf5e7d24b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/a228ea552cde44e88b4de76cf5e7d24b 2024-12-02T17:16:28,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/938c0755f8584596bf33605552ceede0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/938c0755f8584596bf33605552ceede0 2024-12-02T17:16:28,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/ffc5105eb92c46488697f7af4e70db5e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/ffc5105eb92c46488697f7af4e70db5e 2024-12-02T17:16:28,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6ba9a4fdb71a4c71bf2ac85a5bcdbb4e 2024-12-02T17:16:28,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/816fc486131b420ebb31b8e071cb0649 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/816fc486131b420ebb31b8e071cb0649 2024-12-02T17:16:28,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/79a39778e08e4fd18a26542e2d971b8e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/79a39778e08e4fd18a26542e2d971b8e 2024-12-02T17:16:28,911 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3180e148b3214e498c102582ce0bffe8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3180e148b3214e498c102582ce0bffe8 2024-12-02T17:16:28,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3d0efa520a764088b902b25bfc15934f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/3d0efa520a764088b902b25bfc15934f 2024-12-02T17:16:28,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/56ee95fca2b64889bedf8bb55e5b8060 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/56ee95fca2b64889bedf8bb55e5b8060 2024-12-02T17:16:28,914 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f2ee474c06a4cdd9ab7c3e35bbc445c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f2ee474c06a4cdd9ab7c3e35bbc445c 2024-12-02T17:16:28,915 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4e105c4f26594164adaa0486c28225ed to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4e105c4f26594164adaa0486c28225ed 2024-12-02T17:16:28,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9df35d50a06644989c677f864870fdbe to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9df35d50a06644989c677f864870fdbe 2024-12-02T17:16:28,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4059049a7de34c79bd91936c4acaaf0f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4059049a7de34c79bd91936c4acaaf0f 2024-12-02T17:16:28,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ba57345120845388100bb4c9ec7ba24 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/1ba57345120845388100bb4c9ec7ba24 2024-12-02T17:16:28,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0648c64051734add89d5b8b0e373a4be to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0648c64051734add89d5b8b0e373a4be 2024-12-02T17:16:28,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0c2d780871ad461191fda0e0fb9ea7ee to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/0c2d780871ad461191fda0e0fb9ea7ee 2024-12-02T17:16:28,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9af46e551a60476794015ca84daaf3e5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/9af46e551a60476794015ca84daaf3e5 2024-12-02T17:16:29,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/8fa77f16cd214218971deb306e688714 2024-12-02T17:16:29,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/56a6361677eb4e7886bde420727151ae as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae 2024-12-02T17:16:29,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae, entries=150, sequenceid=351, filesize=30.5 K 2024-12-02T17:16:29,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/a9ac37ee0030468f80ef3f9ac1e791ca as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/a9ac37ee0030468f80ef3f9ac1e791ca 2024-12-02T17:16:29,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/a9ac37ee0030468f80ef3f9ac1e791ca, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T17:16:29,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/8fa77f16cd214218971deb306e688714 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/8fa77f16cd214218971deb306e688714 2024-12-02T17:16:29,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/8fa77f16cd214218971deb306e688714, entries=150, sequenceid=351, filesize=12.0 K 2024-12-02T17:16:29,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 213eb7db536967e2cd87656b3cc2847c in 1707ms, sequenceid=351, compaction requested=true 2024-12-02T17:16:29,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 213eb7db536967e2cd87656b3cc2847c:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:29,106 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:29,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:29,106 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:29,107 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:29,107 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/A is initiating minor compaction (all files) 2024-12-02T17:16:29,107 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/A in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:29,107 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/afb8db509d59475987fd628bf555abf5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=92.4 K 2024-12-02T17:16:29,107 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:29,107 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/afb8db509d59475987fd628bf555abf5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae] 2024-12-02T17:16:29,107 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:29,108 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/B is initiating minor compaction (all files) 2024-12-02T17:16:29,108 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/B in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:29,108 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/95b0498023ac4c00abab9b02b371395c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4c347c36749c45f3a1e2b3ee13a7edf9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/45ea45edf6b54282aee8400c04e3f80e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/a9ac37ee0030468f80ef3f9ac1e791ca] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=48.8 K 2024-12-02T17:16:29,108 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb8db509d59475987fd628bf555abf5, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733159785288 2024-12-02T17:16:29,108 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b0498023ac4c00abab9b02b371395c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733159785218 2024-12-02T17:16:29,110 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c347c36749c45f3a1e2b3ee13a7edf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733159785288 2024-12-02T17:16:29,110 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc8befdbe1748ffae75c2ff7eadeecf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733159785553 2024-12-02T17:16:29,111 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a6361677eb4e7886bde420727151ae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733159786215 2024-12-02T17:16:29,111 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ea45edf6b54282aee8400c04e3f80e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733159785553 2024-12-02T17:16:29,114 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ac37ee0030468f80ef3f9ac1e791ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733159786215 2024-12-02T17:16:29,127 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:29,131 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#B#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:29,131 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/805f491e23f84ed1a62d01275be8b1ba is 50, key is test_row_0/B:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:29,142 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202c05bbf1044094724b178c617ba240005_213eb7db536967e2cd87656b3cc2847c store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:29,146 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202c05bbf1044094724b178c617ba240005_213eb7db536967e2cd87656b3cc2847c, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:29,146 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c05bbf1044094724b178c617ba240005_213eb7db536967e2cd87656b3cc2847c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:29,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742334_1510 (size=4469) 2024-12-02T17:16:29,180 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#A#compaction#429 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:29,180 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/67f0ce3da86543caa60aa5e052359401 is 175, key is test_row_0/A:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:29,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742333_1509 (size=12439) 2024-12-02T17:16:29,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742335_1511 (size=31358) 2024-12-02T17:16:29,212 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/67f0ce3da86543caa60aa5e052359401 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/67f0ce3da86543caa60aa5e052359401 2024-12-02T17:16:29,216 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/A of 213eb7db536967e2cd87656b3cc2847c into 67f0ce3da86543caa60aa5e052359401(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:29,216 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:29,216 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/A, priority=13, startTime=1733159789106; duration=0sec 2024-12-02T17:16:29,217 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:29,217 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:A 2024-12-02T17:16:29,217 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:29,218 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:29,218 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 213eb7db536967e2cd87656b3cc2847c/C is initiating minor compaction (all files) 2024-12-02T17:16:29,218 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 213eb7db536967e2cd87656b3cc2847c/C in TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:29,218 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/33ba8bcb95e240ecac485a715da4e44f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/c35119b862fa4deb995b07d691125cf8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6c66f11d845f453a84f28ba99c46e829, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/8fa77f16cd214218971deb306e688714] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp, totalSize=48.8 K 2024-12-02T17:16:29,218 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ba8bcb95e240ecac485a715da4e44f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733159785218 2024-12-02T17:16:29,219 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35119b862fa4deb995b07d691125cf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733159785288 2024-12-02T17:16:29,219 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c66f11d845f453a84f28ba99c46e829, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733159785553 2024-12-02T17:16:29,219 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa77f16cd214218971deb306e688714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733159786215 2024-12-02T17:16:29,246 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 213eb7db536967e2cd87656b3cc2847c#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:29,247 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4f5835ac67384399a36aa0823b1c60c2 is 50, key is test_row_0/C:col10/1733159787397/Put/seqid=0 2024-12-02T17:16:29,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742336_1512 (size=12439) 2024-12-02T17:16:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T17:16:29,347 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:29,603 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/805f491e23f84ed1a62d01275be8b1ba as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/805f491e23f84ed1a62d01275be8b1ba 2024-12-02T17:16:29,607 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/B of 213eb7db536967e2cd87656b3cc2847c into 805f491e23f84ed1a62d01275be8b1ba(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:29,607 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:29,607 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/B, priority=12, startTime=1733159789106; duration=0sec 2024-12-02T17:16:29,607 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:29,607 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:B 2024-12-02T17:16:29,680 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/4f5835ac67384399a36aa0823b1c60c2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f5835ac67384399a36aa0823b1c60c2 2024-12-02T17:16:29,684 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 213eb7db536967e2cd87656b3cc2847c/C of 213eb7db536967e2cd87656b3cc2847c into 4f5835ac67384399a36aa0823b1c60c2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:29,684 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 213eb7db536967e2cd87656b3cc2847c: 2024-12-02T17:16:29,684 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., storeName=213eb7db536967e2cd87656b3cc2847c/C, priority=12, startTime=1733159789106; duration=0sec 2024-12-02T17:16:29,684 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:29,684 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 213eb7db536967e2cd87656b3cc2847c:C 2024-12-02T17:16:30,398 INFO [Thread-1794 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1794. 2024-12-02T17:16:30,398 DEBUG [Thread-1794 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:30,398 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,398 INFO [Thread-1792 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1792. 2024-12-02T17:16:30,398 DEBUG [Thread-1792 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:30,398 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,398 DEBUG [Thread-1792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:30,399 DEBUG [Thread-1792 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:30,418 INFO [Thread-1788 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1788. 2024-12-02T17:16:30,418 DEBUG [Thread-1788 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:30,418 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,419 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 741 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2218 rows 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 728 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2184 rows 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 725 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2173 rows 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 736 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2205 rows 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 734 2024-12-02T17:16:30,419 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2202 rows 2024-12-02T17:16:30,419 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:16:30,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:16:30,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:30,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:30,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T17:16:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T17:16:30,424 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159790423"}]},"ts":"1733159790423"} 2024-12-02T17:16:30,425 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T17:16:30,425 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T17:16:30,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:16:30,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, UNASSIGN}] 2024-12-02T17:16:30,427 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, UNASSIGN 2024-12-02T17:16:30,428 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:30,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, UNASSIGN because future has completed 2024-12-02T17:16:30,429 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:16:30,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T17:16:30,581 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:30,581 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:16:30,581 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 213eb7db536967e2cd87656b3cc2847c, disabling compactions & flushes 2024-12-02T17:16:30,581 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:30,581 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:30,581 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. after waiting 0 ms 2024-12-02T17:16:30,581 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:30,581 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 213eb7db536967e2cd87656b3cc2847c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-02T17:16:30,582 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=A 2024-12-02T17:16:30,582 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:30,582 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=B 2024-12-02T17:16:30,582 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:30,582 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 213eb7db536967e2cd87656b3cc2847c, store=C 2024-12-02T17:16:30,582 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:30,586 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c1849ae40bdc4315afe26b7b6959a598_213eb7db536967e2cd87656b3cc2847c is 50, key is test_row_1/A:col10/1733159790397/Put/seqid=0 2024-12-02T17:16:30,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742337_1513 (size=9914) 2024-12-02T17:16:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T17:16:30,991 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:30,994 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c1849ae40bdc4315afe26b7b6959a598_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c1849ae40bdc4315afe26b7b6959a598_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:30,995 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b1cbda73cefd496d8328d560238f4138, store: [table=TestAcidGuarantees family=A region=213eb7db536967e2cd87656b3cc2847c] 2024-12-02T17:16:30,995 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b1cbda73cefd496d8328d560238f4138 is 175, key is test_row_1/A:col10/1733159790397/Put/seqid=0 2024-12-02T17:16:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742338_1514 (size=22561) 2024-12-02T17:16:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T17:16:31,399 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b1cbda73cefd496d8328d560238f4138 2024-12-02T17:16:31,405 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/27c94d1a955440178778d578c35a2514 is 50, key is test_row_1/B:col10/1733159790397/Put/seqid=0 2024-12-02T17:16:31,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742339_1515 (size=9857) 2024-12-02T17:16:31,408 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/27c94d1a955440178778d578c35a2514 2024-12-02T17:16:31,413 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/37e120c3cf62497c8109a2a466bb975e is 50, key is test_row_1/C:col10/1733159790397/Put/seqid=0 2024-12-02T17:16:31,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742340_1516 (size=9857) 2024-12-02T17:16:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T17:16:31,817 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/37e120c3cf62497c8109a2a466bb975e 2024-12-02T17:16:31,820 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/A/b1cbda73cefd496d8328d560238f4138 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b1cbda73cefd496d8328d560238f4138 2024-12-02T17:16:31,823 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b1cbda73cefd496d8328d560238f4138, entries=100, sequenceid=360, filesize=22.0 K 2024-12-02T17:16:31,824 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/B/27c94d1a955440178778d578c35a2514 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/27c94d1a955440178778d578c35a2514 2024-12-02T17:16:31,827 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/27c94d1a955440178778d578c35a2514, entries=100, sequenceid=360, filesize=9.6 K 2024-12-02T17:16:31,828 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/.tmp/C/37e120c3cf62497c8109a2a466bb975e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/37e120c3cf62497c8109a2a466bb975e 2024-12-02T17:16:31,831 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/37e120c3cf62497c8109a2a466bb975e, entries=100, sequenceid=360, filesize=9.6 K 2024-12-02T17:16:31,831 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 213eb7db536967e2cd87656b3cc2847c in 1250ms, sequenceid=360, compaction requested=false 2024-12-02T17:16:31,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/afb8db509d59475987fd628bf555abf5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae] to archive 2024-12-02T17:16:31,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:31,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/afb8db509d59475987fd628bf555abf5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/afb8db509d59475987fd628bf555abf5 2024-12-02T17:16:31,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/4dc8befdbe1748ffae75c2ff7eadeecf 2024-12-02T17:16:31,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/56a6361677eb4e7886bde420727151ae 2024-12-02T17:16:31,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/95b0498023ac4c00abab9b02b371395c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4c347c36749c45f3a1e2b3ee13a7edf9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/45ea45edf6b54282aee8400c04e3f80e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/a9ac37ee0030468f80ef3f9ac1e791ca] to archive 2024-12-02T17:16:31,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:31,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/95b0498023ac4c00abab9b02b371395c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/95b0498023ac4c00abab9b02b371395c 2024-12-02T17:16:31,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4c347c36749c45f3a1e2b3ee13a7edf9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/4c347c36749c45f3a1e2b3ee13a7edf9 2024-12-02T17:16:31,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/45ea45edf6b54282aee8400c04e3f80e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/45ea45edf6b54282aee8400c04e3f80e 2024-12-02T17:16:31,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/a9ac37ee0030468f80ef3f9ac1e791ca to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/a9ac37ee0030468f80ef3f9ac1e791ca 2024-12-02T17:16:31,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/33ba8bcb95e240ecac485a715da4e44f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/c35119b862fa4deb995b07d691125cf8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6c66f11d845f453a84f28ba99c46e829, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/8fa77f16cd214218971deb306e688714] to archive 2024-12-02T17:16:31,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:16:31,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/33ba8bcb95e240ecac485a715da4e44f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/33ba8bcb95e240ecac485a715da4e44f 2024-12-02T17:16:31,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/c35119b862fa4deb995b07d691125cf8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/c35119b862fa4deb995b07d691125cf8 2024-12-02T17:16:31,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6c66f11d845f453a84f28ba99c46e829 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/6c66f11d845f453a84f28ba99c46e829 2024-12-02T17:16:31,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/8fa77f16cd214218971deb306e688714 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/8fa77f16cd214218971deb306e688714 2024-12-02T17:16:31,850 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/recovered.edits/363.seqid, newMaxSeqId=363, maxSeqId=4 2024-12-02T17:16:31,850 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. 2024-12-02T17:16:31,850 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 213eb7db536967e2cd87656b3cc2847c: Waiting for close lock at 1733159790581Running coprocessor pre-close hooks at 1733159790581Disabling compacts and flushes for region at 1733159790581Disabling writes for close at 1733159790581Obtaining lock to block concurrent updates at 1733159790581Preparing flush snapshotting stores in 213eb7db536967e2cd87656b3cc2847c at 1733159790581Finished memstore snapshotting TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733159790582 (+1 ms)Flushing stores of TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c. at 1733159790582Flushing 213eb7db536967e2cd87656b3cc2847c/A: creating writer at 1733159790582Flushing mob file 213eb7db536967e2cd87656b3cc2847c/A: appending metadata at 1733159790586 (+4 ms)Flushing mob file 213eb7db536967e2cd87656b3cc2847c/A: closing flushed file at 1733159790586Flushing 213eb7db536967e2cd87656b3cc2847c/A: appending metadata at 1733159790995 (+409 ms)Flushing 213eb7db536967e2cd87656b3cc2847c/A: closing flushed file at 1733159790995Flushing 213eb7db536967e2cd87656b3cc2847c/B: creating writer at 1733159791402 (+407 ms)Flushing 213eb7db536967e2cd87656b3cc2847c/B: appending metadata at 1733159791404 (+2 ms)Flushing 213eb7db536967e2cd87656b3cc2847c/B: closing flushed file at 1733159791404Flushing 213eb7db536967e2cd87656b3cc2847c/C: creating writer at 1733159791411 (+7 ms)Flushing 213eb7db536967e2cd87656b3cc2847c/C: appending metadata at 1733159791413 (+2 ms)Flushing 213eb7db536967e2cd87656b3cc2847c/C: closing flushed file at 1733159791413Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bac4b11: reopening flushed file at 1733159791820 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52debf94: reopening flushed file at 1733159791823 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27884dd2: reopening flushed file at 1733159791827 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 213eb7db536967e2cd87656b3cc2847c in 1250ms, sequenceid=360, compaction requested=false at 1733159791832 (+5 ms)Writing region close event to WAL at 1733159791847 (+15 ms)Running coprocessor post-close hooks at 1733159791850 (+3 ms)Closed at 1733159791850 2024-12-02T17:16:31,852 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:31,853 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=213eb7db536967e2cd87656b3cc2847c, regionState=CLOSED 2024-12-02T17:16:31,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:16:31,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-02T17:16:31,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 213eb7db536967e2cd87656b3cc2847c, server=3b3375d1a925,40723,1733159666211 in 1.4260 sec 2024-12-02T17:16:31,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-02T17:16:31,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=213eb7db536967e2cd87656b3cc2847c, UNASSIGN in 1.4310 sec 2024-12-02T17:16:31,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-02T17:16:31,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4340 sec 2024-12-02T17:16:31,861 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159791861"}]},"ts":"1733159791861"} 2024-12-02T17:16:31,863 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T17:16:31,863 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T17:16:31,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4430 sec 2024-12-02T17:16:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-02T17:16:32,567 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:32,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T17:16:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,570 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T17:16:32,570 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,572 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,574 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/recovered.edits] 2024-12-02T17:16:32,576 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/67f0ce3da86543caa60aa5e052359401 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/67f0ce3da86543caa60aa5e052359401 2024-12-02T17:16:32,577 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b1cbda73cefd496d8328d560238f4138 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/A/b1cbda73cefd496d8328d560238f4138 2024-12-02T17:16:32,579 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/27c94d1a955440178778d578c35a2514 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/27c94d1a955440178778d578c35a2514 2024-12-02T17:16:32,580 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/805f491e23f84ed1a62d01275be8b1ba to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/B/805f491e23f84ed1a62d01275be8b1ba 2024-12-02T17:16:32,582 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/37e120c3cf62497c8109a2a466bb975e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/37e120c3cf62497c8109a2a466bb975e 2024-12-02T17:16:32,582 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f5835ac67384399a36aa0823b1c60c2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/C/4f5835ac67384399a36aa0823b1c60c2 2024-12-02T17:16:32,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/recovered.edits/363.seqid to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c/recovered.edits/363.seqid 2024-12-02T17:16:32,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,585 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T17:16:32,585 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T17:16:32,586 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T17:16:32,588 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020dc48d0bc65e426c9a76528aacf5a1a9_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020dc48d0bc65e426c9a76528aacf5a1a9_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,590 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120221e30c860bc84fd3afa44c34d77f8f23_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120221e30c860bc84fd3afa44c34d77f8f23_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,591 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023e8a1eefcacf4aea96209c5e9e878d32_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412023e8a1eefcacf4aea96209c5e9e878d32_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,592 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120256fb216b1fba40aaacc0575968d0d612_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120256fb216b1fba40aaacc0575968d0d612_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,593 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025912ca62928e4308becbb98b43de6f77_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412025912ca62928e4308becbb98b43de6f77_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,594 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026cda91f4379141f4a0f5337f697d84a7_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412026cda91f4379141f4a0f5337f697d84a7_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,595 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202833641cc40da4f5ab3b27f4f111c4790_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202833641cc40da4f5ab3b27f4f111c4790_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,596 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029b45617cb5784851ba994c2a2383751d_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029b45617cb5784851ba994c2a2383751d_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,597 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029d18cd6d75164a3db063fedeadaea742_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029d18cd6d75164a3db063fedeadaea742_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,598 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a86c4c2dd4f6454f93cba145b2852610_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202a86c4c2dd4f6454f93cba145b2852610_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,599 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b8b530629a1846be946975fac2fa9d1d_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202b8b530629a1846be946975fac2fa9d1d_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,599 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bb385d6ca62a4a0e9703e367b1411a8d_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bb385d6ca62a4a0e9703e367b1411a8d_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,600 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bccbe37bcb0749ceaae4382b85621b4f_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202bccbe37bcb0749ceaae4382b85621b4f_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,601 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c0104d99df9c4bb486d876c30fb975be_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c0104d99df9c4bb486d876c30fb975be_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,602 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c1849ae40bdc4315afe26b7b6959a598_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c1849ae40bdc4315afe26b7b6959a598_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,603 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c48cea5819c2415b97ee7b724d6ed843_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c48cea5819c2415b97ee7b724d6ed843_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,604 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cb7048bb27ff4c0d86405f33ba21a77e_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cb7048bb27ff4c0d86405f33ba21a77e_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,605 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eba0b34178be46c6b52f39653240a2f4_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202eba0b34178be46c6b52f39653240a2f4_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,606 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f4acdd63099847f5a075ed42d3cd4d35_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f4acdd63099847f5a075ed42d3cd4d35_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,607 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fe2ecaeddb40452c869a7fcd59ded125_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202fe2ecaeddb40452c869a7fcd59ded125_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,608 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202feef894b261f4f6aa7e4f0f9e7f5b945_213eb7db536967e2cd87656b3cc2847c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202feef894b261f4f6aa7e4f0f9e7f5b945_213eb7db536967e2cd87656b3cc2847c 2024-12-02T17:16:32,608 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T17:16:32,610 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,612 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T17:16:32,615 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T17:16:32,616 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,616 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T17:16:32,616 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733159792616"}]},"ts":"9223372036854775807"} 2024-12-02T17:16:32,618 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T17:16:32,618 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 213eb7db536967e2cd87656b3cc2847c, NAME => 'TestAcidGuarantees,,1733159763613.213eb7db536967e2cd87656b3cc2847c.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T17:16:32,618 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T17:16:32,618 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733159792618"}]},"ts":"9223372036854775807"} 2024-12-02T17:16:32,619 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T17:16:32,620 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-12-02T17:16:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T17:16:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T17:16:32,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:32,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=253 (was 248) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482944666_22 at /127.0.0.1:57692 [Waiting for operation #74] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645024115_22 at /127.0.0.1:57510 [Waiting for operation #127] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645024115_22 at /127.0.0.1:48278 [Waiting for operation #449] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645024115_22 at /127.0.0.1:57670 [Waiting for operation #53] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482944666_22 at /127.0.0.1:57686 [Waiting for operation #70] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/3b3375d1a925:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1014 (was 983) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2483 (was 3207) 2024-12-02T17:16:32,699 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=253, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=1014, ProcessCount=11, AvailableMemoryMB=2483 2024-12-02T17:16:32,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:16:32,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:16:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:32,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:16:32,703 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:32,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-02T17:16:32,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:16:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T17:16:32,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742341_1517 (size=1061) 2024-12-02T17:16:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T17:16:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T17:16:33,110 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:16:33,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742342_1518 (size=53) 2024-12-02T17:16:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T17:16:33,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:16:33,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ccd4d5d4930a7d9225c7192b0068a2e3, disabling compactions & flushes 2024-12-02T17:16:33,515 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. after waiting 0 ms 2024-12-02T17:16:33,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,515 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,515 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ccd4d5d4930a7d9225c7192b0068a2e3: Waiting for close lock at 1733159793515Disabling compacts and flushes for region at 1733159793515Disabling writes for close at 1733159793515Writing region close event to WAL at 1733159793515Closed at 1733159793515 2024-12-02T17:16:33,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:16:33,517 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733159793517"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159793517"}]},"ts":"1733159793517"} 2024-12-02T17:16:33,518 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T17:16:33,519 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:16:33,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159793519"}]},"ts":"1733159793519"} 2024-12-02T17:16:33,520 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T17:16:33,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, ASSIGN}] 2024-12-02T17:16:33,522 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, ASSIGN 2024-12-02T17:16:33,522 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, ASSIGN; state=OFFLINE, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=false 2024-12-02T17:16:33,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=ccd4d5d4930a7d9225c7192b0068a2e3, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:33,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, ASSIGN because future has completed 2024-12-02T17:16:33,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:33,829 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,829 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:16:33,829 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,829 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:16:33,830 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,830 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,831 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,832 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:33,832 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4d5d4930a7d9225c7192b0068a2e3 columnFamilyName A 2024-12-02T17:16:33,832 DEBUG [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:33,832 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.HStore(327): Store=ccd4d5d4930a7d9225c7192b0068a2e3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:33,832 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,833 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:33,833 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4d5d4930a7d9225c7192b0068a2e3 columnFamilyName B 2024-12-02T17:16:33,834 DEBUG [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:33,834 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.HStore(327): Store=ccd4d5d4930a7d9225c7192b0068a2e3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:33,834 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,835 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:16:33,835 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccd4d5d4930a7d9225c7192b0068a2e3 columnFamilyName C 2024-12-02T17:16:33,835 DEBUG [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:16:33,835 INFO [StoreOpener-ccd4d5d4930a7d9225c7192b0068a2e3-1 {}] regionserver.HStore(327): Store=ccd4d5d4930a7d9225c7192b0068a2e3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:16:33,835 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,835 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,836 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,836 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,837 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,837 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T17:16:33,837 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:16:33,838 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,839 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:16:33,840 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened ccd4d5d4930a7d9225c7192b0068a2e3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69065290, jitterRate=0.029153019189834595}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:16:33,840 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:33,840 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for ccd4d5d4930a7d9225c7192b0068a2e3: Running coprocessor pre-open hook at 1733159793830Writing region info on filesystem at 1733159793830Initializing all the Stores at 1733159793830Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159793830Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159793830Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159793830Cleaning up temporary data from old regions at 1733159793837 (+7 ms)Running coprocessor post-open hooks at 1733159793840 (+3 ms)Region opened successfully at 1733159793840 2024-12-02T17:16:33,841 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., pid=120, masterSystemTime=1733159793826 2024-12-02T17:16:33,842 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,842 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:33,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=ccd4d5d4930a7d9225c7192b0068a2e3, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:33,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:16:33,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T17:16:33,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 in 170 msec 2024-12-02T17:16:33,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-02T17:16:33,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, ASSIGN in 325 msec 2024-12-02T17:16:33,848 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:16:33,849 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159793848"}]},"ts":"1733159793848"} 2024-12-02T17:16:33,850 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T17:16:33,850 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:16:33,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-02T17:16:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-02T17:16:34,848 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:34,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,851 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6039fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a239e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,854 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:34,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:34,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:16:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:16:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16beee8 2024-12-02T17:16:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:16:34,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:16:34,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,864 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3820e18e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,869 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31896d6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677dec5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,882 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225c977c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,896 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ee8f87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,904 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21da768f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,909 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3012fee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346ad522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,927 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f345067, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,931 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7838453e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1008d28a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@535b49ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c789e7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,953 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339f2762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73db9bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70e00d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109f3221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:34,981 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d4aac7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:34,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:34,988 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9ed1e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:34,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:16:34,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:16:35,001 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:16:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:16:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:16:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a234fad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:35,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:16:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:16:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:35,005 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:16:35,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3decee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:16:35,007 DEBUG [Thread-2309 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,009 DEBUG [Thread-2309 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,010 DEBUG [Thread-2309 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,013 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:35,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,017 DEBUG [Thread-2294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,017 DEBUG [Thread-2305 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-02T17:16:35,021 DEBUG [Thread-2292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,022 DEBUG [Thread-2303 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,023 DEBUG [Thread-2296 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:35,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:35,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,029 DEBUG [Thread-2301 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:35,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,037 DEBUG [Thread-2290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,037 DEBUG [Thread-2307 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,038 DEBUG [Thread-2290 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,038 DEBUG [Thread-2290 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,049 DEBUG [Thread-2298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:16:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:16:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:16:35,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:16:35,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2] 2024-12-02T17:16:35,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:16:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:35,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:35,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/eae38782e9904a19aa5de95df9781752 is 50, key is test_row_0/A:col10/1733159795064/Put/seqid=0 2024-12-02T17:16:35,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:35,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:35,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742343_1519 (size=12001) 2024-12-02T17:16:35,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159795111, max timestamp=1733159795193Num uniques 859; 2024-12-02T17:16:35,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:35,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159795111, max timestamp=1733159795193Num uniques 859; 2024-12-02T17:16:35,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:35,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159795111, max timestamp=1733159795193Num uniques 859; 2024-12-02T17:16:35,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:35,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/eae38782e9904a19aa5de95df9781752 2024-12-02T17:16:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:35,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:35,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:35,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/11407caaa2cc4033809adcbf8a63b576 is 50, key is test_row_0/B:col10/1733159795064/Put/seqid=0 2024-12-02T17:16:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159855330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159855333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159855357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159855372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159855377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742344_1520 (size=12001) 2024-12-02T17:16:35,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159855477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159855477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159855508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159855509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,523 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:35,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159855517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:35,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:35,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159855707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159855707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159855727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159855727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159855736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:35,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:35,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/11407caaa2cc4033809adcbf8a63b576 2024-12-02T17:16:35,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:35,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:35,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:35,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e65ef2084984402d8ac4e7e1c39371d0 is 50, key is test_row_0/C:col10/1733159795064/Put/seqid=0 2024-12-02T17:16:35,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742345_1521 (size=12001) 2024-12-02T17:16:36,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159856036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159856037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,056 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159856067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159856070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159856069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:36,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e65ef2084984402d8ac4e7e1c39371d0 2024-12-02T17:16:36,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/eae38782e9904a19aa5de95df9781752 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/eae38782e9904a19aa5de95df9781752 2024-12-02T17:16:36,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/eae38782e9904a19aa5de95df9781752, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T17:16:36,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/11407caaa2cc4033809adcbf8a63b576 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/11407caaa2cc4033809adcbf8a63b576 2024-12-02T17:16:36,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/11407caaa2cc4033809adcbf8a63b576, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T17:16:36,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e65ef2084984402d8ac4e7e1c39371d0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e65ef2084984402d8ac4e7e1c39371d0 2024-12-02T17:16:36,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e65ef2084984402d8ac4e7e1c39371d0, entries=150, sequenceid=11, filesize=11.7 K 2024-12-02T17:16:36,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1482ms, sequenceid=11, compaction requested=false 2024-12-02T17:16:36,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:36,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-12-02T17:16:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:16:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:16:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-02T17:16:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:36,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0dd06aa5e2aa4310a31dba8450fad6e3 is 50, key is test_row_0/A:col10/1733159796578/Put/seqid=0 2024-12-02T17:16:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159856613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159856616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159856629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159856636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159856625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742346_1522 (size=16681) 2024-12-02T17:16:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159856768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159856787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159856787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159856787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:36,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:36,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:36,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159856997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159857009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159857017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159857026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0dd06aa5e2aa4310a31dba8450fad6e3 2024-12-02T17:16:37,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/d775835912804f1796624e4d06b97053 is 50, key is test_row_0/B:col10/1733159796578/Put/seqid=0 2024-12-02T17:16:37,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:37,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:37,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:37,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742347_1523 (size=12001) 2024-12-02T17:16:37,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159857328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159857352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159857352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159857357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,405 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:16:37,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/d775835912804f1796624e4d06b97053 2024-12-02T17:16:37,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/197af6b948cd49c7b7bdf9c0da22e585 is 50, key is test_row_0/C:col10/1733159796578/Put/seqid=0 2024-12-02T17:16:37,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:37,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742348_1524 (size=12001) 2024-12-02T17:16:37,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/197af6b948cd49c7b7bdf9c0da22e585 2024-12-02T17:16:37,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0dd06aa5e2aa4310a31dba8450fad6e3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0dd06aa5e2aa4310a31dba8450fad6e3 2024-12-02T17:16:37,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0dd06aa5e2aa4310a31dba8450fad6e3, entries=250, sequenceid=50, filesize=16.3 K 2024-12-02T17:16:37,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/d775835912804f1796624e4d06b97053 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/d775835912804f1796624e4d06b97053 2024-12-02T17:16:37,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:37,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159857647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:37,662 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_1' in ccd4d5d4930a7d9225c7192b0068a2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:37,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/d775835912804f1796624e4d06b97053, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T17:16:37,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/197af6b948cd49c7b7bdf9c0da22e585 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/197af6b948cd49c7b7bdf9c0da22e585 2024-12-02T17:16:37,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/197af6b948cd49c7b7bdf9c0da22e585, entries=150, sequenceid=50, filesize=11.7 K 2024-12-02T17:16:37,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1125ms, sequenceid=50, compaction requested=false 2024-12-02T17:16:37,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:37,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-02T17:16:37,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:37,780 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T17:16:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:37,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c788e917473b4220b416be1ad555e91d is 50, key is test_row_0/A:col10/1733159796616/Put/seqid=0 2024-12-02T17:16:37,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742349_1525 (size=12001) 2024-12-02T17:16:37,865 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c788e917473b4220b416be1ad555e91d 2024-12-02T17:16:37,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:37,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/de6b630043264005a1a56c85072aa44b is 50, key is test_row_0/B:col10/1733159796616/Put/seqid=0 2024-12-02T17:16:37,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742350_1526 (size=12001) 2024-12-02T17:16:38,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159858112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159858113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159858117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159858138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,144 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159858248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159858248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159858248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159858248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/de6b630043264005a1a56c85072aa44b 2024-12-02T17:16:38,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/344fcddc22f647d79285abd2d33638f8 is 50, key is test_row_0/C:col10/1733159796616/Put/seqid=0 2024-12-02T17:16:38,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742351_1527 (size=12001) 2024-12-02T17:16:38,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/344fcddc22f647d79285abd2d33638f8 2024-12-02T17:16:38,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c788e917473b4220b416be1ad555e91d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c788e917473b4220b416be1ad555e91d 2024-12-02T17:16:38,496 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c788e917473b4220b416be1ad555e91d, entries=150, sequenceid=57, filesize=11.7 K 2024-12-02T17:16:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/de6b630043264005a1a56c85072aa44b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/de6b630043264005a1a56c85072aa44b 2024-12-02T17:16:38,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/de6b630043264005a1a56c85072aa44b, entries=150, sequenceid=57, filesize=11.7 K 2024-12-02T17:16:38,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/344fcddc22f647d79285abd2d33638f8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/344fcddc22f647d79285abd2d33638f8 2024-12-02T17:16:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159858477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159858477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159858480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159858487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,531 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/344fcddc22f647d79285abd2d33638f8, entries=150, sequenceid=57, filesize=11.7 K 2024-12-02T17:16:38,537 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=147.60 KB/151140 for ccd4d5d4930a7d9225c7192b0068a2e3 in 758ms, sequenceid=57, compaction requested=true 2024-12-02T17:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-02T17:16:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-02T17:16:38,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T17:16:38,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5140 sec 2024-12-02T17:16:38,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.5290 sec 2024-12-02T17:16:38,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=154.31 KB heapSize=459.66 KB 2024-12-02T17:16:38,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:38,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:38,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:38,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:38,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c509b92b3e314624b58a85adef520c27 is 50, key is test_row_0/A:col10/1733159798862/Put/seqid=0 2024-12-02T17:16:38,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742352_1528 (size=14341) 2024-12-02T17:16:38,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c509b92b3e314624b58a85adef520c27 2024-12-02T17:16:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159858867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/f3a34703dcc4445fbb042e489e0e9e73 is 50, key is test_row_0/B:col10/1733159798862/Put/seqid=0 2024-12-02T17:16:38,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159858897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159858919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159858942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:38,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:39,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742353_1529 (size=12001) 2024-12-02T17:16:39,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/f3a34703dcc4445fbb042e489e0e9e73 2024-12-02T17:16:39,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/2af94e3c399d43d4ab2363cdb3cbf1fd is 50, key is test_row_0/C:col10/1733159798862/Put/seqid=0 2024-12-02T17:16:39,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159859069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:39,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159859098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:39,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159859098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:39,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159859100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T17:16:39,194 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:39,195 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-02T17:16:39,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:39,201 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:39,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T17:16:39,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742354_1530 (size=12001) 2024-12-02T17:16:39,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/2af94e3c399d43d4ab2363cdb3cbf1fd 2024-12-02T17:16:39,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c509b92b3e314624b58a85adef520c27 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c509b92b3e314624b58a85adef520c27 2024-12-02T17:16:39,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c509b92b3e314624b58a85adef520c27, entries=200, sequenceid=83, filesize=14.0 K 2024-12-02T17:16:39,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/f3a34703dcc4445fbb042e489e0e9e73 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f3a34703dcc4445fbb042e489e0e9e73 2024-12-02T17:16:39,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f3a34703dcc4445fbb042e489e0e9e73, entries=150, sequenceid=83, filesize=11.7 K 2024-12-02T17:16:39,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/2af94e3c399d43d4ab2363cdb3cbf1fd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/2af94e3c399d43d4ab2363cdb3cbf1fd 2024-12-02T17:16:39,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/2af94e3c399d43d4ab2363cdb3cbf1fd, entries=150, sequenceid=83, filesize=11.7 K 2024-12-02T17:16:39,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~459.61 KB/470640, currentSize=20.13 KB/20610 for ccd4d5d4930a7d9225c7192b0068a2e3 in 389ms, sequenceid=83, compaction requested=true 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:39,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:16:39,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:39,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:39,259 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:39,259 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:39,259 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:39,259 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/eae38782e9904a19aa5de95df9781752, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0dd06aa5e2aa4310a31dba8450fad6e3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c788e917473b4220b416be1ad555e91d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c509b92b3e314624b58a85adef520c27] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=53.7 K 2024-12-02T17:16:39,259 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae38782e9904a19aa5de95df9781752, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733159795052 2024-12-02T17:16:39,260 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dd06aa5e2aa4310a31dba8450fad6e3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159795328 2024-12-02T17:16:39,260 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c788e917473b4220b416be1ad555e91d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733159796616 2024-12-02T17:16:39,260 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c509b92b3e314624b58a85adef520c27, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733159798094 2024-12-02T17:16:39,268 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:39,268 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:39,268 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:39,268 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e65ef2084984402d8ac4e7e1c39371d0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/197af6b948cd49c7b7bdf9c0da22e585, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/344fcddc22f647d79285abd2d33638f8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/2af94e3c399d43d4ab2363cdb3cbf1fd] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=46.9 K 2024-12-02T17:16:39,270 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e65ef2084984402d8ac4e7e1c39371d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733159795052 2024-12-02T17:16:39,270 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 197af6b948cd49c7b7bdf9c0da22e585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159795349 2024-12-02T17:16:39,271 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 344fcddc22f647d79285abd2d33638f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733159796616 2024-12-02T17:16:39,272 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af94e3c399d43d4ab2363cdb3cbf1fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733159798094 2024-12-02T17:16:39,294 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#447 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:39,295 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c347b09e0f5f4537b88c5cf0529601e7 is 50, key is test_row_0/A:col10/1733159798862/Put/seqid=0 2024-12-02T17:16:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T17:16:39,330 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:39,330 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d6939aae0d944f98a073a6b3ad6c11c7 is 50, key is test_row_0/C:col10/1733159798862/Put/seqid=0 2024-12-02T17:16:39,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-02T17:16:39,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:39,361 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-02T17:16:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:39,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:39,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742355_1531 (size=12139) 2024-12-02T17:16:39,383 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c347b09e0f5f4537b88c5cf0529601e7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c347b09e0f5f4537b88c5cf0529601e7 2024-12-02T17:16:39,388 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into c347b09e0f5f4537b88c5cf0529601e7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:39,388 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:39,388 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=12, startTime=1733159799253; duration=0sec 2024-12-02T17:16:39,389 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:39,389 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:39,389 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:39,390 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:39,390 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:39,390 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:39,390 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/11407caaa2cc4033809adcbf8a63b576, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/d775835912804f1796624e4d06b97053, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/de6b630043264005a1a56c85072aa44b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f3a34703dcc4445fbb042e489e0e9e73] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=46.9 K 2024-12-02T17:16:39,391 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11407caaa2cc4033809adcbf8a63b576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733159795052 2024-12-02T17:16:39,391 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d775835912804f1796624e4d06b97053, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733159795349 2024-12-02T17:16:39,392 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6b630043264005a1a56c85072aa44b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1733159796616 2024-12-02T17:16:39,395 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3a34703dcc4445fbb042e489e0e9e73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733159798094 2024-12-02T17:16:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3e8db422bb814124b6f7d39842b27b58 is 50, key is test_row_0/A:col10/1733159798884/Put/seqid=0 2024-12-02T17:16:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:39,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:39,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742356_1532 (size=12139) 2024-12-02T17:16:39,448 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:39,449 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/71c490775e7549a9b5f74292528b91c5 is 50, key is test_row_0/B:col10/1733159798862/Put/seqid=0 2024-12-02T17:16:39,497 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d6939aae0d944f98a073a6b3ad6c11c7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d6939aae0d944f98a073a6b3ad6c11c7 2024-12-02T17:16:39,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742358_1534 (size=12139) 2024-12-02T17:16:39,508 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into d6939aae0d944f98a073a6b3ad6c11c7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:39,509 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:39,509 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=12, startTime=1733159799253; duration=0sec 2024-12-02T17:16:39,509 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:39,509 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:39,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742357_1533 (size=14341) 2024-12-02T17:16:39,516 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3e8db422bb814124b6f7d39842b27b58 2024-12-02T17:16:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T17:16:39,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ddf6e1953b6047e78f9d0d85c70f6ec3 is 50, key is test_row_0/B:col10/1733159798884/Put/seqid=0 2024-12-02T17:16:39,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742359_1535 (size=12001) 2024-12-02T17:16:39,620 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ddf6e1953b6047e78f9d0d85c70f6ec3 2024-12-02T17:16:39,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a45fcadf8cd44a808f3faff386bfc580 is 50, key is test_row_0/C:col10/1733159798884/Put/seqid=0 2024-12-02T17:16:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742360_1536 (size=12001) 2024-12-02T17:16:39,681 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a45fcadf8cd44a808f3faff386bfc580 2024-12-02T17:16:39,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159799648, max timestamp=1733159799706Num uniques 859; 2024-12-02T17:16:39,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:39,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159799648, max timestamp=1733159799706Num uniques 859; 2024-12-02T17:16:39,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:39,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159799648, max timestamp=1733159799706Num uniques 859; 2024-12-02T17:16:39,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:39,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3e8db422bb814124b6f7d39842b27b58 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3e8db422bb814124b6f7d39842b27b58 2024-12-02T17:16:39,754 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3e8db422bb814124b6f7d39842b27b58, entries=200, sequenceid=91, filesize=14.0 K 2024-12-02T17:16:39,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ddf6e1953b6047e78f9d0d85c70f6ec3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ddf6e1953b6047e78f9d0d85c70f6ec3 2024-12-02T17:16:39,797 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ddf6e1953b6047e78f9d0d85c70f6ec3, entries=150, sequenceid=91, filesize=11.7 K 2024-12-02T17:16:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a45fcadf8cd44a808f3faff386bfc580 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a45fcadf8cd44a808f3faff386bfc580 2024-12-02T17:16:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T17:16:39,853 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a45fcadf8cd44a808f3faff386bfc580, entries=150, sequenceid=91, filesize=11.7 K 2024-12-02T17:16:39,854 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=100.63 KB/103050 for ccd4d5d4930a7d9225c7192b0068a2e3 in 493ms, sequenceid=91, compaction requested=false 2024-12-02T17:16:39,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:39,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:39,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-02T17:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:39,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=107.34 KB heapSize=300.61 KB 2024-12-02T17:16:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-02T17:16:39,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-02T17:16:39,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 675 msec 2024-12-02T17:16:39,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 689 msec 2024-12-02T17:16:39,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0c04ba9d99fb40ebb12f5d46755bbc06 is 50, key is test_row_0/A:col10/1733159799808/Put/seqid=0 2024-12-02T17:16:39,941 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/71c490775e7549a9b5f74292528b91c5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/71c490775e7549a9b5f74292528b91c5 2024-12-02T17:16:39,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742361_1537 (size=19021) 2024-12-02T17:16:39,969 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 71c490775e7549a9b5f74292528b91c5(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:39,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:39,969 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=12, startTime=1733159799253; duration=0sec 2024-12-02T17:16:39,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:39,969 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:39,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159859934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:39,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:39,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159859966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159859972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159859997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159859997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159860104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159860147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159860167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159860167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159860175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T17:16:40,348 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:40,348 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-02T17:16:40,351 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:40,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:40,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:40,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0c04ba9d99fb40ebb12f5d46755bbc06 2024-12-02T17:16:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T17:16:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159860341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/f84390a3ab5c462788db22cecaf1015b is 50, key is test_row_0/B:col10/1733159799808/Put/seqid=0 2024-12-02T17:16:40,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159860393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159860408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742362_1538 (size=12001) 2024-12-02T17:16:40,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/f84390a3ab5c462788db22cecaf1015b 2024-12-02T17:16:40,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159860417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159860423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T17:16:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/8e6d19aed7b44ddd88d354ae0999e106 is 50, key is test_row_0/C:col10/1733159799808/Put/seqid=0 2024-12-02T17:16:40,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T17:16:40,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:40,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:40,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:40,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742363_1539 (size=12001) 2024-12-02T17:16:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T17:16:40,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T17:16:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:40,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159860689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159860737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159860737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159860777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159860781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:40,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T17:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:40,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:40,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/8e6d19aed7b44ddd88d354ae0999e106 2024-12-02T17:16:40,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0c04ba9d99fb40ebb12f5d46755bbc06 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0c04ba9d99fb40ebb12f5d46755bbc06 2024-12-02T17:16:40,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0c04ba9d99fb40ebb12f5d46755bbc06, entries=300, sequenceid=125, filesize=18.6 K 2024-12-02T17:16:40,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/f84390a3ab5c462788db22cecaf1015b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f84390a3ab5c462788db22cecaf1015b 2024-12-02T17:16:40,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f84390a3ab5c462788db22cecaf1015b, entries=150, sequenceid=125, filesize=11.7 K 2024-12-02T17:16:40,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/8e6d19aed7b44ddd88d354ae0999e106 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/8e6d19aed7b44ddd88d354ae0999e106 2024-12-02T17:16:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T17:16:41,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/8e6d19aed7b44ddd88d354ae0999e106, entries=150, sequenceid=125, filesize=11.7 K 2024-12-02T17:16:41,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~300.56 KB/307776, currentSize=73.80 KB/75570 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1144ms, sequenceid=125, compaction requested=true 2024-12-02T17:16:41,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:41,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:41,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:41,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:41,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:41,006 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:41,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:41,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:41,006 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:41,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:41,008 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:41,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:41,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:41,010 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:41,011 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:41,011 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/71c490775e7549a9b5f74292528b91c5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ddf6e1953b6047e78f9d0d85c70f6ec3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f84390a3ab5c462788db22cecaf1015b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=35.3 K 2024-12-02T17:16:41,011 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:41,011 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:41,011 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:41,011 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c347b09e0f5f4537b88c5cf0529601e7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3e8db422bb814124b6f7d39842b27b58, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0c04ba9d99fb40ebb12f5d46755bbc06] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=44.4 K 2024-12-02T17:16:41,012 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c490775e7549a9b5f74292528b91c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733159798094 2024-12-02T17:16:41,012 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c347b09e0f5f4537b88c5cf0529601e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733159798094 2024-12-02T17:16:41,012 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf6e1953b6047e78f9d0d85c70f6ec3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733159798884 2024-12-02T17:16:41,012 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8db422bb814124b6f7d39842b27b58, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733159798884 2024-12-02T17:16:41,012 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting f84390a3ab5c462788db22cecaf1015b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733159799801 2024-12-02T17:16:41,013 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c04ba9d99fb40ebb12f5d46755bbc06, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733159799732 2024-12-02T17:16:41,063 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#457 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:41,063 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/8df16b80b6524a3a84dddb53374d8965 is 50, key is test_row_0/B:col10/1733159799808/Put/seqid=0 2024-12-02T17:16:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/f458fb7ae54346cb992071f2b6af155a is 50, key is test_row_0/A:col10/1733159799964/Put/seqid=0 2024-12-02T17:16:41,084 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:41,085 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/40b33321ad994d7f8aa1c42fd20d8bac is 50, key is test_row_0/A:col10/1733159799808/Put/seqid=0 2024-12-02T17:16:41,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742364_1540 (size=12241) 2024-12-02T17:16:41,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742365_1541 (size=12151) 2024-12-02T17:16:41,207 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/f458fb7ae54346cb992071f2b6af155a 2024-12-02T17:16:41,212 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/8df16b80b6524a3a84dddb53374d8965 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/8df16b80b6524a3a84dddb53374d8965 2024-12-02T17:16:41,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742366_1542 (size=12241) 2024-12-02T17:16:41,225 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 8df16b80b6524a3a84dddb53374d8965(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:41,225 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:41,225 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=13, startTime=1733159801006; duration=0sec 2024-12-02T17:16:41,225 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:41,225 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:41,225 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:41,232 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:41,232 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:41,232 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:41,232 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d6939aae0d944f98a073a6b3ad6c11c7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a45fcadf8cd44a808f3faff386bfc580, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/8e6d19aed7b44ddd88d354ae0999e106] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=35.3 K 2024-12-02T17:16:41,235 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d6939aae0d944f98a073a6b3ad6c11c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733159798094 2024-12-02T17:16:41,236 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a45fcadf8cd44a808f3faff386bfc580, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733159798884 2024-12-02T17:16:41,237 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e6d19aed7b44ddd88d354ae0999e106, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733159799801 2024-12-02T17:16:41,238 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/40b33321ad994d7f8aa1c42fd20d8bac as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/40b33321ad994d7f8aa1c42fd20d8bac 2024-12-02T17:16:41,246 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into 40b33321ad994d7f8aa1c42fd20d8bac(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:41,246 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:41,246 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=13, startTime=1733159801005; duration=0sec 2024-12-02T17:16:41,246 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:41,246 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5ed1991708e84e7c8dd78b925bc936a6 is 50, key is test_row_0/B:col10/1733159799964/Put/seqid=0 2024-12-02T17:16:41,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:41,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:41,268 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:41,269 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b2ae85ed58154b4d92aa2de511e635cd is 50, key is test_row_0/C:col10/1733159799808/Put/seqid=0 2024-12-02T17:16:41,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742367_1543 (size=12151) 2024-12-02T17:16:41,316 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5ed1991708e84e7c8dd78b925bc936a6 2024-12-02T17:16:41,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742368_1544 (size=12241) 2024-12-02T17:16:41,349 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b2ae85ed58154b4d92aa2de511e635cd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b2ae85ed58154b4d92aa2de511e635cd 2024-12-02T17:16:41,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a5cd5de0c2e74524b3f1216684245c29 is 50, key is test_row_0/C:col10/1733159799964/Put/seqid=0 2024-12-02T17:16:41,363 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into b2ae85ed58154b4d92aa2de511e635cd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:41,363 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:41,363 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=13, startTime=1733159801006; duration=0sec 2024-12-02T17:16:41,363 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:41,363 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:41,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742369_1545 (size=12151) 2024-12-02T17:16:41,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159861401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159861396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159861403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159861424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159861431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T17:16:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159861547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159861547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159861557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159861557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159861587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159861787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159861787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,808 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a5cd5de0c2e74524b3f1216684245c29 2024-12-02T17:16:41,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/f458fb7ae54346cb992071f2b6af155a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f458fb7ae54346cb992071f2b6af155a 2024-12-02T17:16:41,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,823 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f458fb7ae54346cb992071f2b6af155a, entries=150, sequenceid=140, filesize=11.9 K 2024-12-02T17:16:41,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5ed1991708e84e7c8dd78b925bc936a6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5ed1991708e84e7c8dd78b925bc936a6 2024-12-02T17:16:41,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159861799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,829 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5ed1991708e84e7c8dd78b925bc936a6, entries=150, sequenceid=140, filesize=11.9 K 2024-12-02T17:16:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a5cd5de0c2e74524b3f1216684245c29 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a5cd5de0c2e74524b3f1216684245c29 2024-12-02T17:16:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159861797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:41,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:41,837 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a5cd5de0c2e74524b3f1216684245c29, entries=150, sequenceid=140, filesize=11.9 K 2024-12-02T17:16:41,838 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4d5d4930a7d9225c7192b0068a2e3 in 830ms, sequenceid=140, compaction requested=false 2024-12-02T17:16:41,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:41,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:41,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-02T17:16:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-02T17:16:41,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:41,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T17:16:41,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4960 sec 2024-12-02T17:16:41,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5040 sec 2024-12-02T17:16:41,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/022d40ecd5844d56b8c3796f62aab80b is 50, key is test_row_0/A:col10/1733159801398/Put/seqid=0 2024-12-02T17:16:41,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742370_1546 (size=14541) 2024-12-02T17:16:41,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/022d40ecd5844d56b8c3796f62aab80b 2024-12-02T17:16:41,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/0bd3079d76fd4f7ba800d3f14b8d207b is 50, key is test_row_0/B:col10/1733159801398/Put/seqid=0 2024-12-02T17:16:41,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742371_1547 (size=12151) 2024-12-02T17:16:42,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159862108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159862116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159862138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159862138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159862140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159862251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159862278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/0bd3079d76fd4f7ba800d3f14b8d207b 2024-12-02T17:16:42,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/61e099e4f16b41de95e5b397d5e97950 is 50, key is test_row_0/C:col10/1733159801398/Put/seqid=0 2024-12-02T17:16:42,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742372_1548 (size=12151) 2024-12-02T17:16:42,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159862489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159862493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T17:16:42,520 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:42,521 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-02T17:16:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T17:16:42,526 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:42,531 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:42,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T17:16:42,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159862647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159862657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T17:16:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159862678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:42,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:42,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:42,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159862817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159862817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T17:16:42,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T17:16:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:42,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:42,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/61e099e4f16b41de95e5b397d5e97950 2024-12-02T17:16:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:42,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/022d40ecd5844d56b8c3796f62aab80b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/022d40ecd5844d56b8c3796f62aab80b 2024-12-02T17:16:42,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/022d40ecd5844d56b8c3796f62aab80b, entries=200, sequenceid=162, filesize=14.2 K 2024-12-02T17:16:42,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/0bd3079d76fd4f7ba800d3f14b8d207b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0bd3079d76fd4f7ba800d3f14b8d207b 2024-12-02T17:16:42,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0bd3079d76fd4f7ba800d3f14b8d207b, entries=150, sequenceid=162, filesize=11.9 K 2024-12-02T17:16:42,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/61e099e4f16b41de95e5b397d5e97950 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/61e099e4f16b41de95e5b397d5e97950 2024-12-02T17:16:42,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/61e099e4f16b41de95e5b397d5e97950, entries=150, sequenceid=162, filesize=11.9 K 2024-12-02T17:16:42,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1075ms, sequenceid=162, compaction requested=true 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:42,924 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:42,925 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:42,925 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:42,925 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:42,925 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/40b33321ad994d7f8aa1c42fd20d8bac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f458fb7ae54346cb992071f2b6af155a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/022d40ecd5844d56b8c3796f62aab80b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=38.0 K 2024-12-02T17:16:42,926 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40b33321ad994d7f8aa1c42fd20d8bac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733159799801 2024-12-02T17:16:42,927 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:42,927 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f458fb7ae54346cb992071f2b6af155a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159799964 2024-12-02T17:16:42,928 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022d40ecd5844d56b8c3796f62aab80b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733159801397 2024-12-02T17:16:42,928 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:42,928 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:42,928 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:42,929 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/8df16b80b6524a3a84dddb53374d8965, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5ed1991708e84e7c8dd78b925bc936a6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0bd3079d76fd4f7ba800d3f14b8d207b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=35.7 K 2024-12-02T17:16:42,929 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df16b80b6524a3a84dddb53374d8965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733159799801 2024-12-02T17:16:42,930 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed1991708e84e7c8dd78b925bc936a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159799964 2024-12-02T17:16:42,930 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd3079d76fd4f7ba800d3f14b8d207b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733159801398 2024-12-02T17:16:42,971 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:42,971 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/29267b4929114570b0dfb2816fad5bb1 is 50, key is test_row_0/B:col10/1733159801398/Put/seqid=0 2024-12-02T17:16:42,975 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:42,975 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/910daac702804055bcf8338b1c932d4e is 50, key is test_row_0/A:col10/1733159801398/Put/seqid=0 2024-12-02T17:16:43,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-02T17:16:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:43,014 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:16:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:43,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:43,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:43,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742374_1550 (size=12493) 2024-12-02T17:16:43,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742373_1549 (size=12493) 2024-12-02T17:16:43,051 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/910daac702804055bcf8338b1c932d4e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/910daac702804055bcf8338b1c932d4e 2024-12-02T17:16:43,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c44213d92dc54a8fa78b9e5df56e676a is 50, key is test_row_0/A:col10/1733159802137/Put/seqid=0 2024-12-02T17:16:43,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742375_1551 (size=12151) 2024-12-02T17:16:43,071 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into 910daac702804055bcf8338b1c932d4e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:43,071 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:43,071 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=13, startTime=1733159802924; duration=0sec 2024-12-02T17:16:43,071 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:43,071 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:43,071 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:43,072 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c44213d92dc54a8fa78b9e5df56e676a 2024-12-02T17:16:43,074 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:43,074 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:43,074 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:43,074 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b2ae85ed58154b4d92aa2de511e635cd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a5cd5de0c2e74524b3f1216684245c29, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/61e099e4f16b41de95e5b397d5e97950] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=35.7 K 2024-12-02T17:16:43,076 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ae85ed58154b4d92aa2de511e635cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733159799801 2024-12-02T17:16:43,078 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5cd5de0c2e74524b3f1216684245c29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1733159799964 2024-12-02T17:16:43,078 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e099e4f16b41de95e5b397d5e97950, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733159801398 2024-12-02T17:16:43,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/20edadfa94bd48adadce0b3940000f40 is 50, key is test_row_0/B:col10/1733159802137/Put/seqid=0 2024-12-02T17:16:43,096 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:43,097 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/96f570d195184c72a17fe4f9501eb25b is 50, key is test_row_0/C:col10/1733159801398/Put/seqid=0 2024-12-02T17:16:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T17:16:43,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742376_1552 (size=12151) 2024-12-02T17:16:43,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742377_1553 (size=12493) 2024-12-02T17:16:43,196 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/96f570d195184c72a17fe4f9501eb25b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/96f570d195184c72a17fe4f9501eb25b 2024-12-02T17:16:43,209 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into 96f570d195184c72a17fe4f9501eb25b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:43,210 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:43,210 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=13, startTime=1733159802924; duration=0sec 2024-12-02T17:16:43,210 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:43,210 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:43,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:43,467 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/29267b4929114570b0dfb2816fad5bb1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/29267b4929114570b0dfb2816fad5bb1 2024-12-02T17:16:43,476 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 29267b4929114570b0dfb2816fad5bb1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:43,476 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:43,476 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=13, startTime=1733159802924; duration=0sec 2024-12-02T17:16:43,476 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:43,476 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159863474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159863475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:43,575 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/20edadfa94bd48adadce0b3940000f40 2024-12-02T17:16:43,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/856d673c75de4e2f85bc5fc2d5e1474d is 50, key is test_row_0/C:col10/1733159802137/Put/seqid=0 2024-12-02T17:16:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159863597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159863597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:43,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742378_1554 (size=12151) 2024-12-02T17:16:43,633 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/856d673c75de4e2f85bc5fc2d5e1474d 2024-12-02T17:16:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c44213d92dc54a8fa78b9e5df56e676a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c44213d92dc54a8fa78b9e5df56e676a 2024-12-02T17:16:43,665 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c44213d92dc54a8fa78b9e5df56e676a, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T17:16:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T17:16:43,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/20edadfa94bd48adadce0b3940000f40 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/20edadfa94bd48adadce0b3940000f40 2024-12-02T17:16:43,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159863669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:43,675 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in ccd4d5d4930a7d9225c7192b0068a2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:43,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:43,682 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/20edadfa94bd48adadce0b3940000f40, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T17:16:43,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/856d673c75de4e2f85bc5fc2d5e1474d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/856d673c75de4e2f85bc5fc2d5e1474d 2024-12-02T17:16:43,691 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/856d673c75de4e2f85bc5fc2d5e1474d, entries=150, sequenceid=175, filesize=11.9 K 2024-12-02T17:16:43,695 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ccd4d5d4930a7d9225c7192b0068a2e3 in 681ms, sequenceid=175, compaction requested=false 2024-12-02T17:16:43,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:43,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:43,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-02T17:16:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-02T17:16:43,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-02T17:16:43,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-12-02T17:16:43,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.1780 sec 2024-12-02T17:16:43,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-02T17:16:43,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:43,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159803444, max timestamp=1733159803712Num uniques 859; 2024-12-02T17:16:43,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:16:43,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159803444, max timestamp=1733159803712Num uniques 859; 2024-12-02T17:16:43,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:16:43,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159803444, max timestamp=1733159803712Num uniques 859; 2024-12-02T17:16:43,731 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:16:43,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:43,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:43,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:43,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3f1cf5815a614f35b83f56c61288d189 is 50, key is test_row_0/A:col10/1733159803712/Put/seqid=0 2024-12-02T17:16:43,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742379_1555 (size=14541) 2024-12-02T17:16:43,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3f1cf5815a614f35b83f56c61288d189 2024-12-02T17:16:43,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ed30b4d3c2454f28b431e92738ac855a is 50, key is test_row_0/B:col10/1733159803712/Put/seqid=0 2024-12-02T17:16:43,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742380_1556 (size=12151) 2024-12-02T17:16:43,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ed30b4d3c2454f28b431e92738ac855a 2024-12-02T17:16:43,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159803856, max timestamp=1733159803886Num uniques 859; 2024-12-02T17:16:43,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:43,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159803856, max timestamp=1733159803886Num uniques 859; 2024-12-02T17:16:43,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:43,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159803856, max timestamp=1733159803886Num uniques 859; 2024-12-02T17:16:43,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:16:43,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/ce37fcec31a940dbb1016e1b84819afa is 50, key is test_row_0/C:col10/1733159803712/Put/seqid=0 2024-12-02T17:16:43,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742381_1557 (size=12151) 2024-12-02T17:16:43,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/ce37fcec31a940dbb1016e1b84819afa 2024-12-02T17:16:43,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3f1cf5815a614f35b83f56c61288d189 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3f1cf5815a614f35b83f56c61288d189 2024-12-02T17:16:43,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3f1cf5815a614f35b83f56c61288d189, entries=200, sequenceid=200, filesize=14.2 K 2024-12-02T17:16:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ed30b4d3c2454f28b431e92738ac855a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ed30b4d3c2454f28b431e92738ac855a 2024-12-02T17:16:44,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733159803977, max timestamp=1733159804010Num uniques 1068; 2024-12-02T17:16:44,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T17:16:44,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733159803977, max timestamp=1733159804010Num uniques 1068; 2024-12-02T17:16:44,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T17:16:44,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733159803977, max timestamp=1733159804010Num uniques 1068; 2024-12-02T17:16:44,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-02T17:16:44,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ed30b4d3c2454f28b431e92738ac855a, entries=150, sequenceid=200, filesize=11.9 K 2024-12-02T17:16:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/ce37fcec31a940dbb1016e1b84819afa as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/ce37fcec31a940dbb1016e1b84819afa 2024-12-02T17:16:44,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/ce37fcec31a940dbb1016e1b84819afa, entries=150, sequenceid=200, filesize=11.9 K 2024-12-02T17:16:44,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=87.22 KB/89310 for ccd4d5d4930a7d9225c7192b0068a2e3 in 339ms, sequenceid=200, compaction requested=true 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:44,065 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:44,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:44,065 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:44,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=93.93 KB heapSize=255.23 KB 2024-12-02T17:16:44,068 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:44,068 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:44,069 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:44,069 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/910daac702804055bcf8338b1c932d4e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c44213d92dc54a8fa78b9e5df56e676a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3f1cf5815a614f35b83f56c61288d189] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=38.3 K 2024-12-02T17:16:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:44,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:16:44,076 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 910daac702804055bcf8338b1c932d4e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733159801398 2024-12-02T17:16:44,076 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:44,076 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:44,076 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:44,076 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/29267b4929114570b0dfb2816fad5bb1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/20edadfa94bd48adadce0b3940000f40, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ed30b4d3c2454f28b431e92738ac855a] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=35.9 K 2024-12-02T17:16:44,077 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29267b4929114570b0dfb2816fad5bb1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733159801398 2024-12-02T17:16:44,077 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c44213d92dc54a8fa78b9e5df56e676a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733159802058 2024-12-02T17:16:44,079 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1cf5815a614f35b83f56c61288d189, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733159803444 2024-12-02T17:16:44,079 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20edadfa94bd48adadce0b3940000f40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733159802058 2024-12-02T17:16:44,080 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed30b4d3c2454f28b431e92738ac855a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733159803444 2024-12-02T17:16:44,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/a30ac3519a7f4f85b58bfab9ef4e239a is 50, key is test_row_0/A:col10/1733159804063/Put/seqid=0 2024-12-02T17:16:44,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742382_1558 (size=16931) 2024-12-02T17:16:44,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/a30ac3519a7f4f85b58bfab9ef4e239a 2024-12-02T17:16:44,134 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#475 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:44,135 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/ef3f5ebb88e644e7822650b3264cca23 is 50, key is test_row_0/A:col10/1733159803712/Put/seqid=0 2024-12-02T17:16:44,146 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:44,146 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ad288de7b59c417ca22b114197d717cb is 50, key is test_row_0/B:col10/1733159803712/Put/seqid=0 2024-12-02T17:16:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159864130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159864138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159864141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159864149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742384_1560 (size=12595) 2024-12-02T17:16:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742383_1559 (size=12595) 2024-12-02T17:16:44,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/1e5e24c9a65a46fd9ed735f067742aac is 50, key is test_row_0/B:col10/1733159804063/Put/seqid=0 2024-12-02T17:16:44,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742385_1561 (size=12151) 2024-12-02T17:16:44,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159864257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159864257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159864280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159864281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159864481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159864481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159864509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159864509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,631 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/ad288de7b59c417ca22b114197d717cb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ad288de7b59c417ca22b114197d717cb 2024-12-02T17:16:44,634 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/ef3f5ebb88e644e7822650b3264cca23 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/ef3f5ebb88e644e7822650b3264cca23 2024-12-02T17:16:44,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/1e5e24c9a65a46fd9ed735f067742aac 2024-12-02T17:16:44,649 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into ad288de7b59c417ca22b114197d717cb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:44,649 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:44,649 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=13, startTime=1733159804065; duration=0sec 2024-12-02T17:16:44,649 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:44,649 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:44,649 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:44,650 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into ef3f5ebb88e644e7822650b3264cca23(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:44,650 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:44,650 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=13, startTime=1733159804065; duration=0sec 2024-12-02T17:16:44,650 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:44,650 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:44,656 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:44,656 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:44,656 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:44,656 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/96f570d195184c72a17fe4f9501eb25b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/856d673c75de4e2f85bc5fc2d5e1474d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/ce37fcec31a940dbb1016e1b84819afa] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=35.9 K 2024-12-02T17:16:44,656 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f570d195184c72a17fe4f9501eb25b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733159801398 2024-12-02T17:16:44,657 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856d673c75de4e2f85bc5fc2d5e1474d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733159802058 2024-12-02T17:16:44,665 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce37fcec31a940dbb1016e1b84819afa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733159803444 2024-12-02T17:16:44,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/422b496b9f344c24bdee3e31a959b5dc is 50, key is test_row_0/C:col10/1733159804063/Put/seqid=0 2024-12-02T17:16:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T17:16:44,689 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:44,691 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-02T17:16:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T17:16:44,694 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:44,695 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:44,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:44,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742386_1562 (size=12151) 2024-12-02T17:16:44,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/422b496b9f344c24bdee3e31a959b5dc 2024-12-02T17:16:44,728 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:44,729 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d289005f407d422da1c5f902ec879469 is 50, key is test_row_0/C:col10/1733159803712/Put/seqid=0 2024-12-02T17:16:44,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/a30ac3519a7f4f85b58bfab9ef4e239a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/a30ac3519a7f4f85b58bfab9ef4e239a 2024-12-02T17:16:44,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/a30ac3519a7f4f85b58bfab9ef4e239a, entries=250, sequenceid=247, filesize=16.5 K 2024-12-02T17:16:44,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/1e5e24c9a65a46fd9ed735f067742aac as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/1e5e24c9a65a46fd9ed735f067742aac 2024-12-02T17:16:44,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742387_1563 (size=12595) 2024-12-02T17:16:44,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T17:16:44,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/1e5e24c9a65a46fd9ed735f067742aac, entries=150, sequenceid=247, filesize=11.9 K 2024-12-02T17:16:44,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/422b496b9f344c24bdee3e31a959b5dc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/422b496b9f344c24bdee3e31a959b5dc 2024-12-02T17:16:44,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159864801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159864801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,825 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d289005f407d422da1c5f902ec879469 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d289005f407d422da1c5f902ec879469 2024-12-02T17:16:44,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/422b496b9f344c24bdee3e31a959b5dc, entries=150, sequenceid=247, filesize=11.9 K 2024-12-02T17:16:44,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~275.11 KB/281712, currentSize=80.51 KB/82440 for ccd4d5d4930a7d9225c7192b0068a2e3 in 774ms, sequenceid=247, compaction requested=false 2024-12-02T17:16:44,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:44,840 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into d289005f407d422da1c5f902ec879469(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:44,840 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:44,840 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=13, startTime=1733159804065; duration=0sec 2024-12-02T17:16:44,840 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:44,840 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:44,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-02T17:16:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:44,857 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:16:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:44,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:44,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/357ac4ab4a3d4bb9b06a4c890907945e is 50, key is test_row_0/A:col10/1733159804144/Put/seqid=0 2024-12-02T17:16:44,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742388_1564 (size=9857) 2024-12-02T17:16:44,929 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/357ac4ab4a3d4bb9b06a4c890907945e 2024-12-02T17:16:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159864981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159864983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T17:16:45,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/57b3a060ef6e4092805917b0bf54b1c2 is 50, key is test_row_0/B:col10/1733159804144/Put/seqid=0 2024-12-02T17:16:45,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742389_1565 (size=9857) 2024-12-02T17:16:45,057 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/57b3a060ef6e4092805917b0bf54b1c2 2024-12-02T17:16:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159865109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159865109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b7801489295745e7890080d8b2cfe88d is 50, key is test_row_0/C:col10/1733159804144/Put/seqid=0 2024-12-02T17:16:45,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742390_1566 (size=9857) 2024-12-02T17:16:45,167 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b7801489295745e7890080d8b2cfe88d 2024-12-02T17:16:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/357ac4ab4a3d4bb9b06a4c890907945e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/357ac4ab4a3d4bb9b06a4c890907945e 2024-12-02T17:16:45,238 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/357ac4ab4a3d4bb9b06a4c890907945e, entries=100, sequenceid=265, filesize=9.6 K 2024-12-02T17:16:45,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/57b3a060ef6e4092805917b0bf54b1c2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/57b3a060ef6e4092805917b0bf54b1c2 2024-12-02T17:16:45,263 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/57b3a060ef6e4092805917b0bf54b1c2, entries=100, sequenceid=265, filesize=9.6 K 2024-12-02T17:16:45,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b7801489295745e7890080d8b2cfe88d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b7801489295745e7890080d8b2cfe88d 2024-12-02T17:16:45,280 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b7801489295745e7890080d8b2cfe88d, entries=100, sequenceid=265, filesize=9.6 K 2024-12-02T17:16:45,283 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ccd4d5d4930a7d9225c7192b0068a2e3 in 426ms, sequenceid=265, compaction requested=true 2024-12-02T17:16:45,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:45,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-02T17:16:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-02T17:16:45,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-02T17:16:45,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 595 msec 2024-12-02T17:16:45,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 610 msec 2024-12-02T17:16:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-02T17:16:45,320 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:45,323 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-02T17:16:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:45,332 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:45,333 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:45,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:45,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0b2a71dde307454582e7a159aa550d5d is 50, key is test_row_0/A:col10/1733159804981/Put/seqid=0 2024-12-02T17:16:45,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742391_1567 (size=14741) 2024-12-02T17:16:45,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0b2a71dde307454582e7a159aa550d5d 2024-12-02T17:16:45,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159865371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159865373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159865376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159865380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/2750a8285ca54c2d92ebab0898cb121f is 50, key is test_row_0/B:col10/1733159804981/Put/seqid=0 2024-12-02T17:16:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:45,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742392_1568 (size=12301) 2024-12-02T17:16:45,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/2750a8285ca54c2d92ebab0898cb121f 2024-12-02T17:16:45,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/75e1c5697af94610928675a68cc7bbf5 is 50, key is test_row_0/C:col10/1733159804981/Put/seqid=0 2024-12-02T17:16:45,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:45,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:45,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742393_1569 (size=12301) 2024-12-02T17:16:45,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159865497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159865517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159865517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159865518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:45,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:45,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:45,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159865688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,706 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in ccd4d5d4930a7d9225c7192b0068a2e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159865727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159865739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159865739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159865748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:45,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/75e1c5697af94610928675a68cc7bbf5 2024-12-02T17:16:45,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0b2a71dde307454582e7a159aa550d5d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0b2a71dde307454582e7a159aa550d5d 2024-12-02T17:16:45,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0b2a71dde307454582e7a159aa550d5d, entries=200, sequenceid=283, filesize=14.4 K 2024-12-02T17:16:45,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/2750a8285ca54c2d92ebab0898cb121f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/2750a8285ca54c2d92ebab0898cb121f 2024-12-02T17:16:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:45,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/2750a8285ca54c2d92ebab0898cb121f, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T17:16:45,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/75e1c5697af94610928675a68cc7bbf5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/75e1c5697af94610928675a68cc7bbf5 2024-12-02T17:16:45,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:45,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/75e1c5697af94610928675a68cc7bbf5, entries=150, sequenceid=283, filesize=12.0 K 2024-12-02T17:16:45,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ccd4d5d4930a7d9225c7192b0068a2e3 in 639ms, sequenceid=283, compaction requested=true 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:45,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:16:45,968 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:45,968 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:45,977 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46904 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:45,977 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:45,977 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,977 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d289005f407d422da1c5f902ec879469, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/422b496b9f344c24bdee3e31a959b5dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b7801489295745e7890080d8b2cfe88d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/75e1c5697af94610928675a68cc7bbf5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=45.8 K 2024-12-02T17:16:45,977 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:45,977 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:45,977 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:45,978 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/ef3f5ebb88e644e7822650b3264cca23, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/a30ac3519a7f4f85b58bfab9ef4e239a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/357ac4ab4a3d4bb9b06a4c890907945e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0b2a71dde307454582e7a159aa550d5d] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=52.9 K 2024-12-02T17:16:45,978 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d289005f407d422da1c5f902ec879469, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733159803444 2024-12-02T17:16:45,981 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 422b496b9f344c24bdee3e31a959b5dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733159804059 2024-12-02T17:16:45,981 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3f5ebb88e644e7822650b3264cca23, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733159803444 2024-12-02T17:16:45,981 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b7801489295745e7890080d8b2cfe88d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733159804139 2024-12-02T17:16:45,981 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a30ac3519a7f4f85b58bfab9ef4e239a, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733159804040 2024-12-02T17:16:45,981 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e1c5697af94610928675a68cc7bbf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733159804941 2024-12-02T17:16:45,982 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357ac4ab4a3d4bb9b06a4c890907945e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733159804139 2024-12-02T17:16:45,982 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2a71dde307454582e7a159aa550d5d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733159804939 2024-12-02T17:16:45,992 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:45,992 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/5694fbea9edc4134b5d7f688149b5d93 is 50, key is test_row_0/C:col10/1733159804981/Put/seqid=0 2024-12-02T17:16:45,996 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:45,997 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0a811a970c764adabab2c7324fc0a40d is 50, key is test_row_0/A:col10/1733159804981/Put/seqid=0 2024-12-02T17:16:46,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742394_1570 (size=12881) 2024-12-02T17:16:46,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742395_1571 (size=12881) 2024-12-02T17:16:46,030 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/0a811a970c764adabab2c7324fc0a40d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0a811a970c764adabab2c7324fc0a40d 2024-12-02T17:16:46,053 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into 0a811a970c764adabab2c7324fc0a40d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:46,053 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:46,053 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=12, startTime=1733159805968; duration=0sec 2024-12-02T17:16:46,053 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:46,053 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:46,053 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:46,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:16:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:46,061 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46904 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:46,062 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:46,062 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,062 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ad288de7b59c417ca22b114197d717cb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/1e5e24c9a65a46fd9ed735f067742aac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/57b3a060ef6e4092805917b0bf54b1c2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/2750a8285ca54c2d92ebab0898cb121f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=45.8 K 2024-12-02T17:16:46,069 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad288de7b59c417ca22b114197d717cb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733159803444 2024-12-02T17:16:46,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/9fbe738e2a7c42ea9fe07a896da7a759 is 50, key is test_row_0/A:col10/1733159806056/Put/seqid=0 2024-12-02T17:16:46,074 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5e24c9a65a46fd9ed735f067742aac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733159804059 2024-12-02T17:16:46,080 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57b3a060ef6e4092805917b0bf54b1c2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733159804139 2024-12-02T17:16:46,080 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2750a8285ca54c2d92ebab0898cb121f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733159804941 2024-12-02T17:16:46,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742396_1572 (size=14741) 2024-12-02T17:16:46,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/9fbe738e2a7c42ea9fe07a896da7a759 2024-12-02T17:16:46,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:46,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:46,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,134 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#489 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:46,135 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5a3339e9f24647a999adec1c11134ad9 is 50, key is test_row_0/B:col10/1733159804981/Put/seqid=0 2024-12-02T17:16:46,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/e62aef24dc7a419f86bb2b3df2a69cb3 is 50, key is test_row_0/B:col10/1733159806056/Put/seqid=0 2024-12-02T17:16:46,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742397_1573 (size=12881) 2024-12-02T17:16:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742398_1574 (size=12301) 2024-12-02T17:16:46,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/e62aef24dc7a419f86bb2b3df2a69cb3 2024-12-02T17:16:46,206 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5a3339e9f24647a999adec1c11134ad9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5a3339e9f24647a999adec1c11134ad9 2024-12-02T17:16:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159866173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159866181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d553ad152e6647a1beae8ec757f8f402 is 50, key is test_row_0/C:col10/1733159806056/Put/seqid=0 2024-12-02T17:16:46,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159866197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159866204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,230 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 5a3339e9f24647a999adec1c11134ad9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:46,230 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:46,230 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=12, startTime=1733159805968; duration=0sec 2024-12-02T17:16:46,230 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:46,230 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:46,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742399_1575 (size=12301) 2024-12-02T17:16:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159866321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159866321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159866327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159866329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,439 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/5694fbea9edc4134b5d7f688149b5d93 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5694fbea9edc4134b5d7f688149b5d93 2024-12-02T17:16:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:46,473 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into 5694fbea9edc4134b5d7f688149b5d93(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:46,473 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:46,474 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=12, startTime=1733159805968; duration=0sec 2024-12-02T17:16:46,474 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:46,474 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159866548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159866557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159866557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159866566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d553ad152e6647a1beae8ec757f8f402 2024-12-02T17:16:46,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/9fbe738e2a7c42ea9fe07a896da7a759 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/9fbe738e2a7c42ea9fe07a896da7a759 2024-12-02T17:16:46,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/9fbe738e2a7c42ea9fe07a896da7a759, entries=200, sequenceid=299, filesize=14.4 K 2024-12-02T17:16:46,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/e62aef24dc7a419f86bb2b3df2a69cb3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/e62aef24dc7a419f86bb2b3df2a69cb3 2024-12-02T17:16:46,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:46,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/e62aef24dc7a419f86bb2b3df2a69cb3, entries=150, sequenceid=299, filesize=12.0 K 2024-12-02T17:16:46,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d553ad152e6647a1beae8ec757f8f402 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d553ad152e6647a1beae8ec757f8f402 2024-12-02T17:16:46,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d553ad152e6647a1beae8ec757f8f402, entries=150, sequenceid=299, filesize=12.0 K 2024-12-02T17:16:46,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ccd4d5d4930a7d9225c7192b0068a2e3 in 743ms, sequenceid=299, compaction requested=false 2024-12-02T17:16:46,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:46,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:46,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:46,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c0971f2140d64d9693c777d9c2103a96 is 50, key is test_row_0/A:col10/1733159806162/Put/seqid=0 2024-12-02T17:16:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742400_1576 (size=14741) 2024-12-02T17:16:46,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:46,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:46,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c0971f2140d64d9693c777d9c2103a96 2024-12-02T17:16:46,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5932e92551f243a9acaebe23478fda5a is 50, key is test_row_0/B:col10/1733159806162/Put/seqid=0 2024-12-02T17:16:46,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159866940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159866946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159866955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:46,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159866955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:46,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742401_1577 (size=12301) 2024-12-02T17:16:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159867067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159867077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159867078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159867087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159867289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159867298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159867298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159867298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5932e92551f243a9acaebe23478fda5a 2024-12-02T17:16:47,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e1bc2fc15f7e403c813e540237b5eee2 is 50, key is test_row_0/C:col10/1733159806162/Put/seqid=0 2024-12-02T17:16:47,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742402_1578 (size=12301) 2024-12-02T17:16:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:47,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159867608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159867608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159867609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159867618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:47,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:47,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e1bc2fc15f7e403c813e540237b5eee2 2024-12-02T17:16:47,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c0971f2140d64d9693c777d9c2103a96 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c0971f2140d64d9693c777d9c2103a96 2024-12-02T17:16:47,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c0971f2140d64d9693c777d9c2103a96, entries=200, sequenceid=319, filesize=14.4 K 2024-12-02T17:16:47,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5932e92551f243a9acaebe23478fda5a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5932e92551f243a9acaebe23478fda5a 2024-12-02T17:16:47,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5932e92551f243a9acaebe23478fda5a, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T17:16:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e1bc2fc15f7e403c813e540237b5eee2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e1bc2fc15f7e403c813e540237b5eee2 2024-12-02T17:16:47,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:47,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:47,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:47,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e1bc2fc15f7e403c813e540237b5eee2, entries=150, sequenceid=319, filesize=12.0 K 2024-12-02T17:16:47,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1034ms, sequenceid=319, compaction requested=true 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:47,913 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:47,913 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:47,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:47,917 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:47,917 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:47,917 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,917 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5a3339e9f24647a999adec1c11134ad9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/e62aef24dc7a419f86bb2b3df2a69cb3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5932e92551f243a9acaebe23478fda5a] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.6 K 2024-12-02T17:16:47,918 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:47,918 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:47,918 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:47,918 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0a811a970c764adabab2c7324fc0a40d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/9fbe738e2a7c42ea9fe07a896da7a759, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c0971f2140d64d9693c777d9c2103a96] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=41.4 K 2024-12-02T17:16:47,918 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a3339e9f24647a999adec1c11134ad9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733159804941 2024-12-02T17:16:47,918 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a811a970c764adabab2c7324fc0a40d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733159804941 2024-12-02T17:16:47,921 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e62aef24dc7a419f86bb2b3df2a69cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733159805369 2024-12-02T17:16:47,921 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fbe738e2a7c42ea9fe07a896da7a759, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733159805369 2024-12-02T17:16:47,921 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5932e92551f243a9acaebe23478fda5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733159806162 2024-12-02T17:16:47,921 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0971f2140d64d9693c777d9c2103a96, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733159806153 2024-12-02T17:16:47,950 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:47,951 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/6c6cd13f801e4c4cbd7439b85f6ef6d4 is 50, key is test_row_0/B:col10/1733159806162/Put/seqid=0 2024-12-02T17:16:47,957 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#496 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:47,958 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/2222c1229b944639a85798ba3fc69dd5 is 50, key is test_row_0/A:col10/1733159806162/Put/seqid=0 2024-12-02T17:16:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742403_1579 (size=12983) 2024-12-02T17:16:48,019 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/6c6cd13f801e4c4cbd7439b85f6ef6d4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/6c6cd13f801e4c4cbd7439b85f6ef6d4 2024-12-02T17:16:48,029 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 6c6cd13f801e4c4cbd7439b85f6ef6d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:48,029 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:48,029 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=13, startTime=1733159807913; duration=0sec 2024-12-02T17:16:48,029 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:48,029 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:48,029 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:48,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742404_1580 (size=12983) 2024-12-02T17:16:48,032 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:48,032 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:48,032 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:48,032 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5694fbea9edc4134b5d7f688149b5d93, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d553ad152e6647a1beae8ec757f8f402, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e1bc2fc15f7e403c813e540237b5eee2] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.6 K 2024-12-02T17:16:48,040 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5694fbea9edc4134b5d7f688149b5d93, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733159804941 2024-12-02T17:16:48,044 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d553ad152e6647a1beae8ec757f8f402, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733159805369 2024-12-02T17:16:48,047 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e1bc2fc15f7e403c813e540237b5eee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733159806162 2024-12-02T17:16:48,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-02T17:16:48,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:48,060 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:16:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:48,075 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/2222c1229b944639a85798ba3fc69dd5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2222c1229b944639a85798ba3fc69dd5 2024-12-02T17:16:48,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/2d52a645565a4b8ab49e5e3cae8b165a is 50, key is test_row_0/A:col10/1733159806936/Put/seqid=0 2024-12-02T17:16:48,099 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:48,099 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d419393db2174f18bb1d6e74cbf7df08 is 50, key is test_row_0/C:col10/1733159806162/Put/seqid=0 2024-12-02T17:16:48,101 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into 2222c1229b944639a85798ba3fc69dd5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:48,101 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:48,101 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=13, startTime=1733159807913; duration=0sec 2024-12-02T17:16:48,101 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:48,101 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:48,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742405_1581 (size=12301) 2024-12-02T17:16:48,138 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/2d52a645565a4b8ab49e5e3cae8b165a 2024-12-02T17:16:48,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:48,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742406_1582 (size=12983) 2024-12-02T17:16:48,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/42791fdc321d4cd5898f27329613e53c is 50, key is test_row_0/B:col10/1733159806936/Put/seqid=0 2024-12-02T17:16:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159868195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159868198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742407_1583 (size=12301) 2024-12-02T17:16:48,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159868201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159868209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159868312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159868321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159868327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159868328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159868537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159868540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159868548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159868548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,566 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/d419393db2174f18bb1d6e74cbf7df08 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d419393db2174f18bb1d6e74cbf7df08 2024-12-02T17:16:48,576 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into d419393db2174f18bb1d6e74cbf7df08(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:48,576 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:48,576 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=13, startTime=1733159807913; duration=0sec 2024-12-02T17:16:48,576 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:48,576 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:48,621 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/42791fdc321d4cd5898f27329613e53c 2024-12-02T17:16:48,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b3347e1978754b4990ff87c286c7633f is 50, key is test_row_0/C:col10/1733159806936/Put/seqid=0 2024-12-02T17:16:48,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742408_1584 (size=12301) 2024-12-02T17:16:48,673 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b3347e1978754b4990ff87c286c7633f 2024-12-02T17:16:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/2d52a645565a4b8ab49e5e3cae8b165a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2d52a645565a4b8ab49e5e3cae8b165a 2024-12-02T17:16:48,701 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2d52a645565a4b8ab49e5e3cae8b165a, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T17:16:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/42791fdc321d4cd5898f27329613e53c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/42791fdc321d4cd5898f27329613e53c 2024-12-02T17:16:48,720 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/42791fdc321d4cd5898f27329613e53c, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T17:16:48,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/b3347e1978754b4990ff87c286c7633f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b3347e1978754b4990ff87c286c7633f 2024-12-02T17:16:48,725 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b3347e1978754b4990ff87c286c7633f, entries=150, sequenceid=335, filesize=12.0 K 2024-12-02T17:16:48,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ccd4d5d4930a7d9225c7192b0068a2e3 in 666ms, sequenceid=335, compaction requested=false 2024-12-02T17:16:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T17:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T17:16:48,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T17:16:48,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3960 sec 2024-12-02T17:16:48,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 3.4110 sec 2024-12-02T17:16:48,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:48,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c1347e8e6be743f898c65b8045e7585b is 50, key is test_row_0/A:col10/1733159808194/Put/seqid=0 2024-12-02T17:16:48,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742409_1585 (size=14741) 2024-12-02T17:16:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159868906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159868909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159868913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159868921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159869029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159869037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159869037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159869047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159869241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159869257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159869257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159869259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c1347e8e6be743f898c65b8045e7585b 2024-12-02T17:16:49,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/a6da9e9317bf46c9ba245428ba3d49a2 is 50, key is test_row_0/B:col10/1733159808194/Put/seqid=0 2024-12-02T17:16:49,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742410_1586 (size=12301) 2024-12-02T17:16:49,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/a6da9e9317bf46c9ba245428ba3d49a2 2024-12-02T17:16:49,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/99c08b7b96fe4745a63f13f84e732e64 is 50, key is test_row_0/C:col10/1733159808194/Put/seqid=0 2024-12-02T17:16:49,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742411_1587 (size=12301) 2024-12-02T17:16:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T17:16:49,490 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:49,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-02T17:16:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T17:16:49,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:49,500 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:49,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:49,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159869572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T17:16:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159869587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159869587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159869624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T17:16:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:49,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:49,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45162 deadline: 1733159869729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:49,743 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_0' in ccd4d5d4930a7d9225c7192b0068a2e3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8347 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:16:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T17:16:49,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T17:16:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:49,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/99c08b7b96fe4745a63f13f84e732e64 2024-12-02T17:16:49,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/c1347e8e6be743f898c65b8045e7585b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c1347e8e6be743f898c65b8045e7585b 2024-12-02T17:16:49,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c1347e8e6be743f898c65b8045e7585b, entries=200, sequenceid=355, filesize=14.4 K 2024-12-02T17:16:49,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/a6da9e9317bf46c9ba245428ba3d49a2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/a6da9e9317bf46c9ba245428ba3d49a2 2024-12-02T17:16:49,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/a6da9e9317bf46c9ba245428ba3d49a2, entries=150, sequenceid=355, filesize=12.0 K 2024-12-02T17:16:49,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/99c08b7b96fe4745a63f13f84e732e64 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/99c08b7b96fe4745a63f13f84e732e64 2024-12-02T17:16:49,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/99c08b7b96fe4745a63f13f84e732e64, entries=150, sequenceid=355, filesize=12.0 K 2024-12-02T17:16:49,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1043ms, sequenceid=355, compaction requested=true 2024-12-02T17:16:49,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:49,904 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:49,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:49,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:49,904 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:49,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:49,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:49,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:49,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:49,905 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:49,905 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:49,905 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,905 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2222c1229b944639a85798ba3fc69dd5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2d52a645565a4b8ab49e5e3cae8b165a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c1347e8e6be743f898c65b8045e7585b] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=39.1 K 2024-12-02T17:16:49,906 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:49,906 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2222c1229b944639a85798ba3fc69dd5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733159806162 2024-12-02T17:16:49,906 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:49,906 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,906 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/6c6cd13f801e4c4cbd7439b85f6ef6d4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/42791fdc321d4cd5898f27329613e53c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/a6da9e9317bf46c9ba245428ba3d49a2] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.7 K 2024-12-02T17:16:49,906 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d52a645565a4b8ab49e5e3cae8b165a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159806936 2024-12-02T17:16:49,906 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1347e8e6be743f898c65b8045e7585b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733159808194 2024-12-02T17:16:49,906 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6cd13f801e4c4cbd7439b85f6ef6d4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733159806162 2024-12-02T17:16:49,907 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 42791fdc321d4cd5898f27329613e53c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159806936 2024-12-02T17:16:49,907 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting a6da9e9317bf46c9ba245428ba3d49a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733159808194 2024-12-02T17:16:49,933 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:49,934 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/f65960a3874742bdb367962050e0c063 is 50, key is test_row_0/A:col10/1733159808194/Put/seqid=0 2024-12-02T17:16:49,947 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:49,947 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/60ef4be0c8ce4e35a1d4f977d9ebe191 is 50, key is test_row_0/B:col10/1733159808194/Put/seqid=0 2024-12-02T17:16:49,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-02T17:16:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:49,981 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:49,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742412_1588 (size=13085) 2024-12-02T17:16:49,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742413_1589 (size=13085) 2024-12-02T17:16:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/133577ca87f14c5d991acb5cc95d333c is 50, key is test_row_0/A:col10/1733159808907/Put/seqid=0 2024-12-02T17:16:50,020 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/60ef4be0c8ce4e35a1d4f977d9ebe191 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/60ef4be0c8ce4e35a1d4f977d9ebe191 2024-12-02T17:16:50,033 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 60ef4be0c8ce4e35a1d4f977d9ebe191(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:50,033 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:50,033 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=13, startTime=1733159809904; duration=0sec 2024-12-02T17:16:50,033 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:50,033 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:50,033 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:50,036 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:50,036 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:50,036 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:50,036 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d419393db2174f18bb1d6e74cbf7df08, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b3347e1978754b4990ff87c286c7633f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/99c08b7b96fe4745a63f13f84e732e64] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.7 K 2024-12-02T17:16:50,038 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d419393db2174f18bb1d6e74cbf7df08, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733159806162 2024-12-02T17:16:50,039 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b3347e1978754b4990ff87c286c7633f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733159806936 2024-12-02T17:16:50,039 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c08b7b96fe4745a63f13f84e732e64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733159808194 2024-12-02T17:16:50,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742414_1590 (size=12301) 2024-12-02T17:16:50,051 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/133577ca87f14c5d991acb5cc95d333c 2024-12-02T17:16:50,083 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:50,084 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/79144efab3454d64a3eb1a15afbfc43c is 50, key is test_row_0/C:col10/1733159808194/Put/seqid=0 2024-12-02T17:16:50,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/bdb1e5afb311426fbaabc964d915c1c6 is 50, key is test_row_0/B:col10/1733159808907/Put/seqid=0 2024-12-02T17:16:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:50,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T17:16:50,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742416_1592 (size=12301) 2024-12-02T17:16:50,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742415_1591 (size=13085) 2024-12-02T17:16:50,155 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/79144efab3454d64a3eb1a15afbfc43c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/79144efab3454d64a3eb1a15afbfc43c 2024-12-02T17:16:50,161 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into 79144efab3454d64a3eb1a15afbfc43c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:50,161 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:50,161 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=13, startTime=1733159809905; duration=0sec 2024-12-02T17:16:50,161 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:50,161 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:50,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159870215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159870215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159870220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159870221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159870337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159870337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159870339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159870347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,412 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/f65960a3874742bdb367962050e0c063 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f65960a3874742bdb367962050e0c063 2024-12-02T17:16:50,455 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into f65960a3874742bdb367962050e0c063(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:50,455 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:50,456 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=13, startTime=1733159809904; duration=0sec 2024-12-02T17:16:50,456 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:50,456 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:50,546 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/bdb1e5afb311426fbaabc964d915c1c6 2024-12-02T17:16:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159870557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159870558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159870569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/49efa7da6b6648c5a494d849454cf0e1 is 50, key is test_row_0/C:col10/1733159808907/Put/seqid=0 2024-12-02T17:16:50,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159870571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T17:16:50,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742417_1593 (size=12301) 2024-12-02T17:16:50,644 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/49efa7da6b6648c5a494d849454cf0e1 2024-12-02T17:16:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/133577ca87f14c5d991acb5cc95d333c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/133577ca87f14c5d991acb5cc95d333c 2024-12-02T17:16:50,685 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/133577ca87f14c5d991acb5cc95d333c, entries=150, sequenceid=369, filesize=12.0 K 2024-12-02T17:16:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/bdb1e5afb311426fbaabc964d915c1c6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/bdb1e5afb311426fbaabc964d915c1c6 2024-12-02T17:16:50,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/bdb1e5afb311426fbaabc964d915c1c6, entries=150, sequenceid=369, filesize=12.0 K 2024-12-02T17:16:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/49efa7da6b6648c5a494d849454cf0e1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/49efa7da6b6648c5a494d849454cf0e1 2024-12-02T17:16:50,769 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/49efa7da6b6648c5a494d849454cf0e1, entries=150, sequenceid=369, filesize=12.0 K 2024-12-02T17:16:50,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4d5d4930a7d9225c7192b0068a2e3 in 798ms, sequenceid=369, compaction requested=false 2024-12-02T17:16:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-02T17:16:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-02T17:16:50,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T17:16:50,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2880 sec 2024-12-02T17:16:50,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3050 sec 2024-12-02T17:16:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:50,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:16:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:50,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:50,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:50,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:50,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:50,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:50,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/d5d37dd623de4837bee761156aacad18 is 50, key is test_row_0/A:col10/1733159810903/Put/seqid=0 2024-12-02T17:16:50,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742418_1594 (size=12301) 2024-12-02T17:16:50,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159870965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159870971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:50,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:50,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159870987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159870987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159871101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159871104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159871107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159871107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159871317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159871328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159871328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159871333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/d5d37dd623de4837bee761156aacad18 2024-12-02T17:16:51,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/c4015713caba48ff80f13b0b03f48321 is 50, key is test_row_0/B:col10/1733159810903/Put/seqid=0 2024-12-02T17:16:51,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742419_1595 (size=12301) 2024-12-02T17:16:51,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/c4015713caba48ff80f13b0b03f48321 2024-12-02T17:16:51,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e22832b64cb3452da600f519b669a25f is 50, key is test_row_0/C:col10/1733159810903/Put/seqid=0 2024-12-02T17:16:51,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742420_1596 (size=12301) 2024-12-02T17:16:51,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e22832b64cb3452da600f519b669a25f 2024-12-02T17:16:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/d5d37dd623de4837bee761156aacad18 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/d5d37dd623de4837bee761156aacad18 2024-12-02T17:16:51,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/d5d37dd623de4837bee761156aacad18, entries=150, sequenceid=392, filesize=12.0 K 2024-12-02T17:16:51,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/c4015713caba48ff80f13b0b03f48321 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/c4015713caba48ff80f13b0b03f48321 2024-12-02T17:16:51,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/c4015713caba48ff80f13b0b03f48321, entries=150, sequenceid=392, filesize=12.0 K 2024-12-02T17:16:51,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/e22832b64cb3452da600f519b669a25f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e22832b64cb3452da600f519b669a25f 2024-12-02T17:16:51,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e22832b64cb3452da600f519b669a25f, entries=150, sequenceid=392, filesize=12.0 K 2024-12-02T17:16:51,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ccd4d5d4930a7d9225c7192b0068a2e3 in 668ms, sequenceid=392, compaction requested=true 2024-12-02T17:16:51,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:51,577 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:51,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:51,580 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:51,580 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:51,580 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:51,580 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,581 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f65960a3874742bdb367962050e0c063, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/133577ca87f14c5d991acb5cc95d333c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/d5d37dd623de4837bee761156aacad18] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.8 K 2024-12-02T17:16:51,582 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65960a3874742bdb367962050e0c063, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733159808194 2024-12-02T17:16:51,582 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 133577ca87f14c5d991acb5cc95d333c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733159808900 2024-12-02T17:16:51,583 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:51,583 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:51,583 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,583 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/60ef4be0c8ce4e35a1d4f977d9ebe191, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/bdb1e5afb311426fbaabc964d915c1c6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/c4015713caba48ff80f13b0b03f48321] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.8 K 2024-12-02T17:16:51,585 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ef4be0c8ce4e35a1d4f977d9ebe191, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733159808194 2024-12-02T17:16:51,585 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5d37dd623de4837bee761156aacad18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733159810213 2024-12-02T17:16:51,585 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb1e5afb311426fbaabc964d915c1c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733159808900 2024-12-02T17:16:51,586 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c4015713caba48ff80f13b0b03f48321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733159810213 2024-12-02T17:16:51,606 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:51,607 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/6c4b3ec2666641f988c942dc23ce077c is 50, key is test_row_0/A:col10/1733159810903/Put/seqid=0 2024-12-02T17:16:51,613 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:51,613 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5aa25186b1e2498bb3687d556f93da62 is 50, key is test_row_0/B:col10/1733159810903/Put/seqid=0 2024-12-02T17:16:51,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:16:51,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:51,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:51,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:51,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:51,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:51,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T17:16:51,649 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:51,649 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-02T17:16:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T17:16:51,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:51,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742421_1597 (size=13187) 2024-12-02T17:16:51,654 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:51,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:51,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/6c09b736e80f43c4b2a1a6bd1203ae19 is 50, key is test_row_0/A:col10/1733159810982/Put/seqid=0 2024-12-02T17:16:51,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742423_1599 (size=14741) 2024-12-02T17:16:51,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/6c09b736e80f43c4b2a1a6bd1203ae19 2024-12-02T17:16:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742422_1598 (size=13187) 2024-12-02T17:16:51,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/0682024944e44470be9d1f1a2a30d8a8 is 50, key is test_row_0/B:col10/1733159810982/Put/seqid=0 2024-12-02T17:16:51,751 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/5aa25186b1e2498bb3687d556f93da62 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5aa25186b1e2498bb3687d556f93da62 2024-12-02T17:16:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T17:16:51,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742424_1600 (size=12301) 2024-12-02T17:16:51,765 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into 5aa25186b1e2498bb3687d556f93da62(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:51,765 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:51,765 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=13, startTime=1733159811577; duration=0sec 2024-12-02T17:16:51,766 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:51,766 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:51,766 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:16:51,770 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:16:51,770 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:51,770 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,770 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/79144efab3454d64a3eb1a15afbfc43c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/49efa7da6b6648c5a494d849454cf0e1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e22832b64cb3452da600f519b669a25f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=36.8 K 2024-12-02T17:16:51,773 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 79144efab3454d64a3eb1a15afbfc43c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733159808194 2024-12-02T17:16:51,773 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 49efa7da6b6648c5a494d849454cf0e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1733159808900 2024-12-02T17:16:51,778 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting e22832b64cb3452da600f519b669a25f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733159810213 2024-12-02T17:16:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159871750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159871756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159871776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159871781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:51,808 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:51,811 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/0970cbf8030347a5ad2e650f6efe7a89 is 50, key is test_row_0/C:col10/1733159810903/Put/seqid=0 2024-12-02T17:16:51,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742425_1601 (size=13187) 2024-12-02T17:16:51,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159871887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159871901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159871906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159871907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:51,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:51,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:51,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:51,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:51,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:51,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T17:16:52,077 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/6c4b3ec2666641f988c942dc23ce077c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c4b3ec2666641f988c942dc23ce077c 2024-12-02T17:16:52,109 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into 6c4b3ec2666641f988c942dc23ce077c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:52,109 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:52,109 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=13, startTime=1733159811577; duration=0sec 2024-12-02T17:16:52,109 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:52,109 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:52,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159872118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159872119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159872119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159872129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/0682024944e44470be9d1f1a2a30d8a8 2024-12-02T17:16:52,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/48232d9f184f4495af014c5059fcb55e is 50, key is test_row_0/C:col10/1733159810982/Put/seqid=0 2024-12-02T17:16:52,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742426_1602 (size=12301) 2024-12-02T17:16:52,258 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/0970cbf8030347a5ad2e650f6efe7a89 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/0970cbf8030347a5ad2e650f6efe7a89 2024-12-02T17:16:52,266 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into 0970cbf8030347a5ad2e650f6efe7a89(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:52,266 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:52,266 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=13, startTime=1733159811577; duration=0sec 2024-12-02T17:16:52,266 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:52,266 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:52,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T17:16:52,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159872456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159872457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159872458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159872467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:52,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:52,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:52,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/48232d9f184f4495af014c5059fcb55e 2024-12-02T17:16:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/6c09b736e80f43c4b2a1a6bd1203ae19 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c09b736e80f43c4b2a1a6bd1203ae19 2024-12-02T17:16:52,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:52,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c09b736e80f43c4b2a1a6bd1203ae19, entries=200, sequenceid=406, filesize=14.4 K 2024-12-02T17:16:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/0682024944e44470be9d1f1a2a30d8a8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0682024944e44470be9d1f1a2a30d8a8 2024-12-02T17:16:52,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0682024944e44470be9d1f1a2a30d8a8, entries=150, sequenceid=406, filesize=12.0 K 2024-12-02T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/48232d9f184f4495af014c5059fcb55e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/48232d9f184f4495af014c5059fcb55e 2024-12-02T17:16:52,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/48232d9f184f4495af014c5059fcb55e, entries=150, sequenceid=406, filesize=12.0 K 2024-12-02T17:16:52,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1144ms, sequenceid=406, compaction requested=false 2024-12-02T17:16:52,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T17:16:52,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-02T17:16:52,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:52,899 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:16:52,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:52,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:52,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:52,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:52,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3bf73de3c1d146c29117686b245b7671 is 50, key is test_row_0/A:col10/1733159811777/Put/seqid=0 2024-12-02T17:16:52,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742427_1603 (size=12301) 2024-12-02T17:16:52,950 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3bf73de3c1d146c29117686b245b7671 2024-12-02T17:16:52,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/78d0378be72443728f0d7b4419a0d996 is 50, key is test_row_0/B:col10/1733159811777/Put/seqid=0 2024-12-02T17:16:52,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:53,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742428_1604 (size=12301) 2024-12-02T17:16:53,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159873028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159873033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159873033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159873055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159873157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159873157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159873169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159873181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159873377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159873380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159873398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159873400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,427 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/78d0378be72443728f0d7b4419a0d996 2024-12-02T17:16:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/81e6fd77672a474181a6c84bab4717f8 is 50, key is test_row_0/C:col10/1733159811777/Put/seqid=0 2024-12-02T17:16:53,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742429_1605 (size=12301) 2024-12-02T17:16:53,484 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/81e6fd77672a474181a6c84bab4717f8 2024-12-02T17:16:53,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/3bf73de3c1d146c29117686b245b7671 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3bf73de3c1d146c29117686b245b7671 2024-12-02T17:16:53,520 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3bf73de3c1d146c29117686b245b7671, entries=150, sequenceid=427, filesize=12.0 K 2024-12-02T17:16:53,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/78d0378be72443728f0d7b4419a0d996 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/78d0378be72443728f0d7b4419a0d996 2024-12-02T17:16:53,526 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/78d0378be72443728f0d7b4419a0d996, entries=150, sequenceid=427, filesize=12.0 K 2024-12-02T17:16:53,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/81e6fd77672a474181a6c84bab4717f8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/81e6fd77672a474181a6c84bab4717f8 2024-12-02T17:16:53,538 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/81e6fd77672a474181a6c84bab4717f8, entries=150, sequenceid=427, filesize=12.0 K 2024-12-02T17:16:53,539 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ccd4d5d4930a7d9225c7192b0068a2e3 in 640ms, sequenceid=427, compaction requested=true 2024-12-02T17:16:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:53,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-02T17:16:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-02T17:16:53,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T17:16:53,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8930 sec 2024-12-02T17:16:53,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.9000 sec 2024-12-02T17:16:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:53,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-02T17:16:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:53,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:53,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:53,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:53,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:53,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/b8f6488706e347229fb02a002be4f544 is 50, key is test_row_0/A:col10/1733159813053/Put/seqid=0 2024-12-02T17:16:53,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742430_1606 (size=12301) 2024-12-02T17:16:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159873768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159873783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159873785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159873789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T17:16:53,813 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:53,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:16:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-02T17:16:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T17:16:53,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:16:53,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:16:53,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:16:53,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159873897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159873897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T17:16:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159873919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159873919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:53,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:53,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:53,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T17:16:54,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159874128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159874128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159874147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159874157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/b8f6488706e347229fb02a002be4f544 2024-12-02T17:16:54,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/96f0d74fc68d4f2da6a75d9bd3ba2854 is 50, key is test_row_0/B:col10/1733159813053/Put/seqid=0 2024-12-02T17:16:54,246 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T17:16:54,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742431_1607 (size=12301) 2024-12-02T17:16:54,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/96f0d74fc68d4f2da6a75d9bd3ba2854 2024-12-02T17:16:54,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:54,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/099832a26b8a482b94c1a71aef7808d0 is 50, key is test_row_0/C:col10/1733159813053/Put/seqid=0 2024-12-02T17:16:54,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742432_1608 (size=12301) 2024-12-02T17:16:54,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/099832a26b8a482b94c1a71aef7808d0 2024-12-02T17:16:54,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/b8f6488706e347229fb02a002be4f544 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/b8f6488706e347229fb02a002be4f544 2024-12-02T17:16:54,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/b8f6488706e347229fb02a002be4f544, entries=150, sequenceid=443, filesize=12.0 K 2024-12-02T17:16:54,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/96f0d74fc68d4f2da6a75d9bd3ba2854 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/96f0d74fc68d4f2da6a75d9bd3ba2854 2024-12-02T17:16:54,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/96f0d74fc68d4f2da6a75d9bd3ba2854, entries=150, sequenceid=443, filesize=12.0 K 2024-12-02T17:16:54,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/099832a26b8a482b94c1a71aef7808d0 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/099832a26b8a482b94c1a71aef7808d0 2024-12-02T17:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T17:16:54,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/099832a26b8a482b94c1a71aef7808d0, entries=150, sequenceid=443, filesize=12.0 K 2024-12-02T17:16:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ccd4d5d4930a7d9225c7192b0068a2e3 in 747ms, sequenceid=443, compaction requested=true 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ccd4d5d4930a7d9225c7192b0068a2e3:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:16:54,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:16:54,458 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:54,458 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:54,459 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:54,459 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/A is initiating minor compaction (all files) 2024-12-02T17:16:54,460 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/A in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,460 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c4b3ec2666641f988c942dc23ce077c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c09b736e80f43c4b2a1a6bd1203ae19, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3bf73de3c1d146c29117686b245b7671, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/b8f6488706e347229fb02a002be4f544] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=51.3 K 2024-12-02T17:16:54,460 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:54,460 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/C is initiating minor compaction (all files) 2024-12-02T17:16:54,460 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/C in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,460 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/0970cbf8030347a5ad2e650f6efe7a89, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/48232d9f184f4495af014c5059fcb55e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/81e6fd77672a474181a6c84bab4717f8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/099832a26b8a482b94c1a71aef7808d0] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=48.9 K 2024-12-02T17:16:54,460 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c4b3ec2666641f988c942dc23ce077c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733159810213 2024-12-02T17:16:54,461 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0970cbf8030347a5ad2e650f6efe7a89, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733159810213 2024-12-02T17:16:54,461 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c09b736e80f43c4b2a1a6bd1203ae19, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159810922 2024-12-02T17:16:54,461 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 48232d9f184f4495af014c5059fcb55e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159810922 2024-12-02T17:16:54,461 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf73de3c1d146c29117686b245b7671, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733159811747 2024-12-02T17:16:54,461 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e6fd77672a474181a6c84bab4717f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733159811747 2024-12-02T17:16:54,461 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8f6488706e347229fb02a002be4f544, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733159813053 2024-12-02T17:16:54,462 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 099832a26b8a482b94c1a71aef7808d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733159813053 2024-12-02T17:16:54,479 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#C#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:54,480 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a2d7abfb9d8f4a7bb017ef96b000fef6 is 50, key is test_row_0/C:col10/1733159813053/Put/seqid=0 2024-12-02T17:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:54,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-02T17:16:54,486 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#A#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:54,486 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/e14473ea6b234b6c8c9c9b0e68192074 is 50, key is test_row_0/A:col10/1733159813053/Put/seqid=0 2024-12-02T17:16:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:54,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742433_1609 (size=13323) 2024-12-02T17:16:54,515 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/a2d7abfb9d8f4a7bb017ef96b000fef6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a2d7abfb9d8f4a7bb017ef96b000fef6 2024-12-02T17:16:54,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/8a4e4393449a448ba2eec6bc43855f11 is 50, key is test_row_0/A:col10/1733159813784/Put/seqid=0 2024-12-02T17:16:54,527 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/C of ccd4d5d4930a7d9225c7192b0068a2e3 into a2d7abfb9d8f4a7bb017ef96b000fef6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:54,527 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:54,527 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/C, priority=12, startTime=1733159814458; duration=0sec 2024-12-02T17:16:54,527 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:16:54,527 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:C 2024-12-02T17:16:54,527 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:16:54,529 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:16:54,529 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): ccd4d5d4930a7d9225c7192b0068a2e3/B is initiating minor compaction (all files) 2024-12-02T17:16:54,529 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccd4d5d4930a7d9225c7192b0068a2e3/B in TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,529 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5aa25186b1e2498bb3687d556f93da62, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0682024944e44470be9d1f1a2a30d8a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/78d0378be72443728f0d7b4419a0d996, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/96f0d74fc68d4f2da6a75d9bd3ba2854] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp, totalSize=48.9 K 2024-12-02T17:16:54,529 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa25186b1e2498bb3687d556f93da62, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733159810213 2024-12-02T17:16:54,530 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0682024944e44470be9d1f1a2a30d8a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733159810922 2024-12-02T17:16:54,530 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d0378be72443728f0d7b4419a0d996, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1733159811747 2024-12-02T17:16:54,530 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f0d74fc68d4f2da6a75d9bd3ba2854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733159813053 2024-12-02T17:16:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742434_1610 (size=13323) 2024-12-02T17:16:54,546 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccd4d5d4930a7d9225c7192b0068a2e3#B#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:16:54,547 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/b3ccb2418c7c41d3846fde8070566051 is 50, key is test_row_0/B:col10/1733159813053/Put/seqid=0 2024-12-02T17:16:54,548 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/e14473ea6b234b6c8c9c9b0e68192074 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/e14473ea6b234b6c8c9c9b0e68192074 2024-12-02T17:16:54,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742435_1611 (size=14741) 2024-12-02T17:16:54,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/8a4e4393449a448ba2eec6bc43855f11 2024-12-02T17:16:54,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742436_1612 (size=13323) 2024-12-02T17:16:54,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/fe6ff23ef65e4eaabe54b468d1520855 is 50, key is test_row_0/B:col10/1733159813784/Put/seqid=0 2024-12-02T17:16:54,573 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/b3ccb2418c7c41d3846fde8070566051 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/b3ccb2418c7c41d3846fde8070566051 2024-12-02T17:16:54,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742437_1613 (size=12301) 2024-12-02T17:16:54,597 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/A of ccd4d5d4930a7d9225c7192b0068a2e3 into e14473ea6b234b6c8c9c9b0e68192074(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:54,597 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ccd4d5d4930a7d9225c7192b0068a2e3/B of ccd4d5d4930a7d9225c7192b0068a2e3 into b3ccb2418c7c41d3846fde8070566051(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:16:54,597 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:54,597 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:54,597 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/A, priority=12, startTime=1733159814458; duration=0sec 2024-12-02T17:16:54,597 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., storeName=ccd4d5d4930a7d9225c7192b0068a2e3/B, priority=12, startTime=1733159814458; duration=0sec 2024-12-02T17:16:54,597 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:54,597 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:A 2024-12-02T17:16:54,597 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:16:54,597 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccd4d5d4930a7d9225c7192b0068a2e3:B 2024-12-02T17:16:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159874549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159874598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159874598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159874598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159874707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159874726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159874726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159874727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:54,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:54,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:54,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:16:54,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45234 deadline: 1733159874927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45194 deadline: 1733159874937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45220 deadline: 1733159874937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:16:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45160 deadline: 1733159874947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 , the old value is region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:16:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., hostname=3b3375d1a925,40723,1733159666211, seqNum=2 because the exception is null or not the one we care about 2024-12-02T17:16:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T17:16:54,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/fe6ff23ef65e4eaabe54b468d1520855 2024-12-02T17:16:54,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/5ed116df3f3a4779b910e80552b4220b is 50, key is test_row_0/C:col10/1733159813784/Put/seqid=0 2024-12-02T17:16:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742438_1614 (size=12301) 2024-12-02T17:16:55,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/5ed116df3f3a4779b910e80552b4220b 2024-12-02T17:16:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/8a4e4393449a448ba2eec6bc43855f11 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/8a4e4393449a448ba2eec6bc43855f11 2024-12-02T17:16:55,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/8a4e4393449a448ba2eec6bc43855f11, entries=200, sequenceid=461, filesize=14.4 K 2024-12-02T17:16:55,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/fe6ff23ef65e4eaabe54b468d1520855 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/fe6ff23ef65e4eaabe54b468d1520855 2024-12-02T17:16:55,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/fe6ff23ef65e4eaabe54b468d1520855, entries=150, sequenceid=461, filesize=12.0 K 2024-12-02T17:16:55,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/5ed116df3f3a4779b910e80552b4220b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5ed116df3f3a4779b910e80552b4220b 2024-12-02T17:16:55,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5ed116df3f3a4779b910e80552b4220b, entries=150, sequenceid=461, filesize=12.0 K 2024-12-02T17:16:55,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for ccd4d5d4930a7d9225c7192b0068a2e3 in 536ms, sequenceid=461, compaction requested=false 2024-12-02T17:16:55,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:55,051 INFO [Thread-2309 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2309. 2024-12-02T17:16:55,051 DEBUG [Thread-2309 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,051 DEBUG [Thread-2309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,052 DEBUG [Thread-2309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,053 INFO [Thread-2301 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2301. 2024-12-02T17:16:55,053 DEBUG [Thread-2301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,053 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,053 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,055 INFO [Thread-2305 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2305. 2024-12-02T17:16:55,055 DEBUG [Thread-2305 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,055 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,055 DEBUG [Thread-2305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,059 INFO [Thread-2303 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2303. 2024-12-02T17:16:55,059 DEBUG [Thread-2303 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,059 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,059 DEBUG [Thread-2303 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,061 INFO [Thread-2307 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2307. 2024-12-02T17:16:55,061 DEBUG [Thread-2307 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,061 DEBUG [Thread-2307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,061 DEBUG [Thread-2307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-02T17:16:55,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:55,070 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-02T17:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:55,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:55,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/12cb2d86dcc84fbb90df2914993577b5 is 50, key is test_row_0/A:col10/1733159814570/Put/seqid=0 2024-12-02T17:16:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742439_1615 (size=12301) 2024-12-02T17:16:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:55,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. as already flushing 2024-12-02T17:16:55,249 INFO [Thread-2298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2298. 2024-12-02T17:16:55,249 DEBUG [Thread-2298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,249 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,249 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,258 INFO [Thread-2296 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2296. 2024-12-02T17:16:55,258 DEBUG [Thread-2296 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,258 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,258 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,259 INFO [Thread-2294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2294. 2024-12-02T17:16:55,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,259 DEBUG [Thread-2294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,259 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,259 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,260 INFO [Thread-2290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2290. 2024-12-02T17:16:55,260 DEBUG [Thread-2290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:55,260 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,260 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:55,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:55,477 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/12cb2d86dcc84fbb90df2914993577b5 2024-12-02T17:16:55,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/99031365bc20436aa086193b00a960d7 is 50, key is test_row_0/B:col10/1733159814570/Put/seqid=0 2024-12-02T17:16:55,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742440_1616 (size=12301) 2024-12-02T17:16:55,886 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/99031365bc20436aa086193b00a960d7 2024-12-02T17:16:55,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/9249b77c393f4236831ed040048c3075 is 50, key is test_row_0/C:col10/1733159814570/Put/seqid=0 2024-12-02T17:16:55,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742441_1617 (size=12301) 2024-12-02T17:16:55,894 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/9249b77c393f4236831ed040048c3075 2024-12-02T17:16:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/12cb2d86dcc84fbb90df2914993577b5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/12cb2d86dcc84fbb90df2914993577b5 2024-12-02T17:16:55,900 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/12cb2d86dcc84fbb90df2914993577b5, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T17:16:55,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/99031365bc20436aa086193b00a960d7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/99031365bc20436aa086193b00a960d7 2024-12-02T17:16:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/99031365bc20436aa086193b00a960d7, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T17:16:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/9249b77c393f4236831ed040048c3075 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/9249b77c393f4236831ed040048c3075 2024-12-02T17:16:55,906 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/9249b77c393f4236831ed040048c3075, entries=150, sequenceid=480, filesize=12.0 K 2024-12-02T17:16:55,907 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for ccd4d5d4930a7d9225c7192b0068a2e3 in 837ms, sequenceid=480, compaction requested=true 2024-12-02T17:16:55,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for ccd4d5d4930a7d9225c7192b0068a2e3: 2024-12-02T17:16:55,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:55,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-02T17:16:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-02T17:16:55,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-02T17:16:55,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0850 sec 2024-12-02T17:16:55,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.0930 sec 2024-12-02T17:16:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T17:16:55,967 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:16:59,789 INFO [Thread-2292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2292. 2024-12-02T17:16:59,789 DEBUG [Thread-2292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:16:59,789 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:59,789 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:59,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 528 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1584 rows 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 532 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1594 rows 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 530 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1589 rows 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 531 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1592 rows 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 533 2024-12-02T17:16:59,789 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1591 rows 2024-12-02T17:16:59,789 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:16:59,789 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:16:59,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:59,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:16:59,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:16:59,790 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T17:16:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T17:16:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T17:16:59,793 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159819793"}]},"ts":"1733159819793"} 2024-12-02T17:16:59,794 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T17:16:59,794 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T17:16:59,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:16:59,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, UNASSIGN}] 2024-12-02T17:16:59,796 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, UNASSIGN 2024-12-02T17:16:59,797 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=ccd4d5d4930a7d9225c7192b0068a2e3, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:16:59,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, UNASSIGN because future has completed 2024-12-02T17:16:59,798 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:16:59,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:16:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T17:16:59,950 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing ccd4d5d4930a7d9225c7192b0068a2e3, disabling compactions & flushes 2024-12-02T17:16:59,951 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. after waiting 0 ms 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:16:59,951 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing ccd4d5d4930a7d9225c7192b0068a2e3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=A 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=B 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ccd4d5d4930a7d9225c7192b0068a2e3, store=C 2024-12-02T17:16:59,951 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:16:59,955 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/17c19a1c66414b8782499f0fa8ad19c4 is 50, key is test_row_0/A:col10/1733159819787/Put/seqid=0 2024-12-02T17:16:59,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742442_1618 (size=9857) 2024-12-02T17:17:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T17:17:00,359 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/17c19a1c66414b8782499f0fa8ad19c4 2024-12-02T17:17:00,365 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/59239a09559a463297dd79bc7a68014c is 50, key is test_row_0/B:col10/1733159819787/Put/seqid=0 2024-12-02T17:17:00,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742443_1619 (size=9857) 2024-12-02T17:17:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T17:17:00,768 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/59239a09559a463297dd79bc7a68014c 2024-12-02T17:17:00,774 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/36ab71efea5940c6be105f806d876eac is 50, key is test_row_0/C:col10/1733159819787/Put/seqid=0 2024-12-02T17:17:00,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742444_1620 (size=9857) 2024-12-02T17:17:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T17:17:01,177 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/36ab71efea5940c6be105f806d876eac 2024-12-02T17:17:01,181 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/A/17c19a1c66414b8782499f0fa8ad19c4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/17c19a1c66414b8782499f0fa8ad19c4 2024-12-02T17:17:01,184 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/17c19a1c66414b8782499f0fa8ad19c4, entries=100, sequenceid=488, filesize=9.6 K 2024-12-02T17:17:01,185 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/B/59239a09559a463297dd79bc7a68014c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/59239a09559a463297dd79bc7a68014c 2024-12-02T17:17:01,188 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/59239a09559a463297dd79bc7a68014c, entries=100, sequenceid=488, filesize=9.6 K 2024-12-02T17:17:01,189 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/.tmp/C/36ab71efea5940c6be105f806d876eac as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/36ab71efea5940c6be105f806d876eac 2024-12-02T17:17:01,191 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/36ab71efea5940c6be105f806d876eac, entries=100, sequenceid=488, filesize=9.6 K 2024-12-02T17:17:01,192 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1241ms, sequenceid=488, compaction requested=true 2024-12-02T17:17:01,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/eae38782e9904a19aa5de95df9781752, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0dd06aa5e2aa4310a31dba8450fad6e3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c788e917473b4220b416be1ad555e91d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c509b92b3e314624b58a85adef520c27, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c347b09e0f5f4537b88c5cf0529601e7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3e8db422bb814124b6f7d39842b27b58, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0c04ba9d99fb40ebb12f5d46755bbc06, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/40b33321ad994d7f8aa1c42fd20d8bac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f458fb7ae54346cb992071f2b6af155a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/022d40ecd5844d56b8c3796f62aab80b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/910daac702804055bcf8338b1c932d4e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c44213d92dc54a8fa78b9e5df56e676a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3f1cf5815a614f35b83f56c61288d189, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/ef3f5ebb88e644e7822650b3264cca23, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/a30ac3519a7f4f85b58bfab9ef4e239a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/357ac4ab4a3d4bb9b06a4c890907945e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0b2a71dde307454582e7a159aa550d5d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0a811a970c764adabab2c7324fc0a40d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/9fbe738e2a7c42ea9fe07a896da7a759, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c0971f2140d64d9693c777d9c2103a96, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2222c1229b944639a85798ba3fc69dd5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2d52a645565a4b8ab49e5e3cae8b165a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c1347e8e6be743f898c65b8045e7585b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f65960a3874742bdb367962050e0c063, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/133577ca87f14c5d991acb5cc95d333c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c4b3ec2666641f988c942dc23ce077c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/d5d37dd623de4837bee761156aacad18, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c09b736e80f43c4b2a1a6bd1203ae19, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3bf73de3c1d146c29117686b245b7671, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/b8f6488706e347229fb02a002be4f544] to archive 2024-12-02T17:17:01,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:17:01,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/eae38782e9904a19aa5de95df9781752 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/eae38782e9904a19aa5de95df9781752 2024-12-02T17:17:01,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0dd06aa5e2aa4310a31dba8450fad6e3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0dd06aa5e2aa4310a31dba8450fad6e3 2024-12-02T17:17:01,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c788e917473b4220b416be1ad555e91d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c788e917473b4220b416be1ad555e91d 2024-12-02T17:17:01,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c509b92b3e314624b58a85adef520c27 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c509b92b3e314624b58a85adef520c27 2024-12-02T17:17:01,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c347b09e0f5f4537b88c5cf0529601e7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c347b09e0f5f4537b88c5cf0529601e7 2024-12-02T17:17:01,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3e8db422bb814124b6f7d39842b27b58 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3e8db422bb814124b6f7d39842b27b58 2024-12-02T17:17:01,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0c04ba9d99fb40ebb12f5d46755bbc06 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0c04ba9d99fb40ebb12f5d46755bbc06 2024-12-02T17:17:01,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/40b33321ad994d7f8aa1c42fd20d8bac to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/40b33321ad994d7f8aa1c42fd20d8bac 2024-12-02T17:17:01,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f458fb7ae54346cb992071f2b6af155a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f458fb7ae54346cb992071f2b6af155a 2024-12-02T17:17:01,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/022d40ecd5844d56b8c3796f62aab80b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/022d40ecd5844d56b8c3796f62aab80b 2024-12-02T17:17:01,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/910daac702804055bcf8338b1c932d4e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/910daac702804055bcf8338b1c932d4e 2024-12-02T17:17:01,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c44213d92dc54a8fa78b9e5df56e676a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c44213d92dc54a8fa78b9e5df56e676a 2024-12-02T17:17:01,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3f1cf5815a614f35b83f56c61288d189 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3f1cf5815a614f35b83f56c61288d189 2024-12-02T17:17:01,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/ef3f5ebb88e644e7822650b3264cca23 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/ef3f5ebb88e644e7822650b3264cca23 2024-12-02T17:17:01,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/a30ac3519a7f4f85b58bfab9ef4e239a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/a30ac3519a7f4f85b58bfab9ef4e239a 2024-12-02T17:17:01,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/357ac4ab4a3d4bb9b06a4c890907945e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/357ac4ab4a3d4bb9b06a4c890907945e 2024-12-02T17:17:01,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0b2a71dde307454582e7a159aa550d5d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0b2a71dde307454582e7a159aa550d5d 2024-12-02T17:17:01,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0a811a970c764adabab2c7324fc0a40d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/0a811a970c764adabab2c7324fc0a40d 2024-12-02T17:17:01,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/9fbe738e2a7c42ea9fe07a896da7a759 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/9fbe738e2a7c42ea9fe07a896da7a759 2024-12-02T17:17:01,209 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c0971f2140d64d9693c777d9c2103a96 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c0971f2140d64d9693c777d9c2103a96 2024-12-02T17:17:01,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2222c1229b944639a85798ba3fc69dd5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2222c1229b944639a85798ba3fc69dd5 2024-12-02T17:17:01,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2d52a645565a4b8ab49e5e3cae8b165a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/2d52a645565a4b8ab49e5e3cae8b165a 2024-12-02T17:17:01,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c1347e8e6be743f898c65b8045e7585b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/c1347e8e6be743f898c65b8045e7585b 2024-12-02T17:17:01,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f65960a3874742bdb367962050e0c063 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/f65960a3874742bdb367962050e0c063 2024-12-02T17:17:01,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/133577ca87f14c5d991acb5cc95d333c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/133577ca87f14c5d991acb5cc95d333c 2024-12-02T17:17:01,214 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c4b3ec2666641f988c942dc23ce077c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c4b3ec2666641f988c942dc23ce077c 2024-12-02T17:17:01,215 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/d5d37dd623de4837bee761156aacad18 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/d5d37dd623de4837bee761156aacad18 2024-12-02T17:17:01,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c09b736e80f43c4b2a1a6bd1203ae19 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/6c09b736e80f43c4b2a1a6bd1203ae19 2024-12-02T17:17:01,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3bf73de3c1d146c29117686b245b7671 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/3bf73de3c1d146c29117686b245b7671 2024-12-02T17:17:01,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/b8f6488706e347229fb02a002be4f544 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/b8f6488706e347229fb02a002be4f544 2024-12-02T17:17:01,219 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/11407caaa2cc4033809adcbf8a63b576, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/d775835912804f1796624e4d06b97053, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/de6b630043264005a1a56c85072aa44b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/71c490775e7549a9b5f74292528b91c5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f3a34703dcc4445fbb042e489e0e9e73, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ddf6e1953b6047e78f9d0d85c70f6ec3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/8df16b80b6524a3a84dddb53374d8965, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f84390a3ab5c462788db22cecaf1015b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5ed1991708e84e7c8dd78b925bc936a6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/29267b4929114570b0dfb2816fad5bb1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0bd3079d76fd4f7ba800d3f14b8d207b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/20edadfa94bd48adadce0b3940000f40, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ad288de7b59c417ca22b114197d717cb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ed30b4d3c2454f28b431e92738ac855a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/1e5e24c9a65a46fd9ed735f067742aac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/57b3a060ef6e4092805917b0bf54b1c2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5a3339e9f24647a999adec1c11134ad9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/2750a8285ca54c2d92ebab0898cb121f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/e62aef24dc7a419f86bb2b3df2a69cb3, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/6c6cd13f801e4c4cbd7439b85f6ef6d4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5932e92551f243a9acaebe23478fda5a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/42791fdc321d4cd5898f27329613e53c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/60ef4be0c8ce4e35a1d4f977d9ebe191, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/a6da9e9317bf46c9ba245428ba3d49a2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/bdb1e5afb311426fbaabc964d915c1c6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5aa25186b1e2498bb3687d556f93da62, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/c4015713caba48ff80f13b0b03f48321, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0682024944e44470be9d1f1a2a30d8a8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/78d0378be72443728f0d7b4419a0d996, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/96f0d74fc68d4f2da6a75d9bd3ba2854] to archive 2024-12-02T17:17:01,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:17:01,221 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/11407caaa2cc4033809adcbf8a63b576 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/11407caaa2cc4033809adcbf8a63b576 2024-12-02T17:17:01,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/d775835912804f1796624e4d06b97053 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/d775835912804f1796624e4d06b97053 2024-12-02T17:17:01,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/de6b630043264005a1a56c85072aa44b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/de6b630043264005a1a56c85072aa44b 2024-12-02T17:17:01,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/71c490775e7549a9b5f74292528b91c5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/71c490775e7549a9b5f74292528b91c5 2024-12-02T17:17:01,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f3a34703dcc4445fbb042e489e0e9e73 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f3a34703dcc4445fbb042e489e0e9e73 2024-12-02T17:17:01,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ddf6e1953b6047e78f9d0d85c70f6ec3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ddf6e1953b6047e78f9d0d85c70f6ec3 2024-12-02T17:17:01,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/8df16b80b6524a3a84dddb53374d8965 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/8df16b80b6524a3a84dddb53374d8965 2024-12-02T17:17:01,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f84390a3ab5c462788db22cecaf1015b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/f84390a3ab5c462788db22cecaf1015b 2024-12-02T17:17:01,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5ed1991708e84e7c8dd78b925bc936a6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5ed1991708e84e7c8dd78b925bc936a6 2024-12-02T17:17:01,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/29267b4929114570b0dfb2816fad5bb1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/29267b4929114570b0dfb2816fad5bb1 2024-12-02T17:17:01,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0bd3079d76fd4f7ba800d3f14b8d207b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0bd3079d76fd4f7ba800d3f14b8d207b 2024-12-02T17:17:01,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/20edadfa94bd48adadce0b3940000f40 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/20edadfa94bd48adadce0b3940000f40 2024-12-02T17:17:01,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ad288de7b59c417ca22b114197d717cb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ad288de7b59c417ca22b114197d717cb 2024-12-02T17:17:01,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ed30b4d3c2454f28b431e92738ac855a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/ed30b4d3c2454f28b431e92738ac855a 2024-12-02T17:17:01,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/1e5e24c9a65a46fd9ed735f067742aac to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/1e5e24c9a65a46fd9ed735f067742aac 2024-12-02T17:17:01,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/57b3a060ef6e4092805917b0bf54b1c2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/57b3a060ef6e4092805917b0bf54b1c2 2024-12-02T17:17:01,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5a3339e9f24647a999adec1c11134ad9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5a3339e9f24647a999adec1c11134ad9 2024-12-02T17:17:01,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/2750a8285ca54c2d92ebab0898cb121f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/2750a8285ca54c2d92ebab0898cb121f 2024-12-02T17:17:01,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/e62aef24dc7a419f86bb2b3df2a69cb3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/e62aef24dc7a419f86bb2b3df2a69cb3 2024-12-02T17:17:01,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/6c6cd13f801e4c4cbd7439b85f6ef6d4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/6c6cd13f801e4c4cbd7439b85f6ef6d4 2024-12-02T17:17:01,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5932e92551f243a9acaebe23478fda5a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5932e92551f243a9acaebe23478fda5a 2024-12-02T17:17:01,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/42791fdc321d4cd5898f27329613e53c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/42791fdc321d4cd5898f27329613e53c 2024-12-02T17:17:01,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/60ef4be0c8ce4e35a1d4f977d9ebe191 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/60ef4be0c8ce4e35a1d4f977d9ebe191 2024-12-02T17:17:01,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/a6da9e9317bf46c9ba245428ba3d49a2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/a6da9e9317bf46c9ba245428ba3d49a2 2024-12-02T17:17:01,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/bdb1e5afb311426fbaabc964d915c1c6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/bdb1e5afb311426fbaabc964d915c1c6 2024-12-02T17:17:01,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5aa25186b1e2498bb3687d556f93da62 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/5aa25186b1e2498bb3687d556f93da62 2024-12-02T17:17:01,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/c4015713caba48ff80f13b0b03f48321 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/c4015713caba48ff80f13b0b03f48321 2024-12-02T17:17:01,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0682024944e44470be9d1f1a2a30d8a8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/0682024944e44470be9d1f1a2a30d8a8 2024-12-02T17:17:01,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/78d0378be72443728f0d7b4419a0d996 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/78d0378be72443728f0d7b4419a0d996 2024-12-02T17:17:01,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/96f0d74fc68d4f2da6a75d9bd3ba2854 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/96f0d74fc68d4f2da6a75d9bd3ba2854 2024-12-02T17:17:01,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e65ef2084984402d8ac4e7e1c39371d0, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/197af6b948cd49c7b7bdf9c0da22e585, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/344fcddc22f647d79285abd2d33638f8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d6939aae0d944f98a073a6b3ad6c11c7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/2af94e3c399d43d4ab2363cdb3cbf1fd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a45fcadf8cd44a808f3faff386bfc580, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b2ae85ed58154b4d92aa2de511e635cd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/8e6d19aed7b44ddd88d354ae0999e106, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a5cd5de0c2e74524b3f1216684245c29, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/96f570d195184c72a17fe4f9501eb25b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/61e099e4f16b41de95e5b397d5e97950, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/856d673c75de4e2f85bc5fc2d5e1474d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d289005f407d422da1c5f902ec879469, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/ce37fcec31a940dbb1016e1b84819afa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/422b496b9f344c24bdee3e31a959b5dc, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b7801489295745e7890080d8b2cfe88d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5694fbea9edc4134b5d7f688149b5d93, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/75e1c5697af94610928675a68cc7bbf5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d553ad152e6647a1beae8ec757f8f402, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d419393db2174f18bb1d6e74cbf7df08, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e1bc2fc15f7e403c813e540237b5eee2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b3347e1978754b4990ff87c286c7633f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/79144efab3454d64a3eb1a15afbfc43c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/99c08b7b96fe4745a63f13f84e732e64, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/49efa7da6b6648c5a494d849454cf0e1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/0970cbf8030347a5ad2e650f6efe7a89, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e22832b64cb3452da600f519b669a25f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/48232d9f184f4495af014c5059fcb55e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/81e6fd77672a474181a6c84bab4717f8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/099832a26b8a482b94c1a71aef7808d0] to archive 2024-12-02T17:17:01,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:17:01,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e65ef2084984402d8ac4e7e1c39371d0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e65ef2084984402d8ac4e7e1c39371d0 2024-12-02T17:17:01,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/197af6b948cd49c7b7bdf9c0da22e585 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/197af6b948cd49c7b7bdf9c0da22e585 2024-12-02T17:17:01,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/344fcddc22f647d79285abd2d33638f8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/344fcddc22f647d79285abd2d33638f8 2024-12-02T17:17:01,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d6939aae0d944f98a073a6b3ad6c11c7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d6939aae0d944f98a073a6b3ad6c11c7 2024-12-02T17:17:01,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/2af94e3c399d43d4ab2363cdb3cbf1fd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/2af94e3c399d43d4ab2363cdb3cbf1fd 2024-12-02T17:17:01,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a45fcadf8cd44a808f3faff386bfc580 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a45fcadf8cd44a808f3faff386bfc580 2024-12-02T17:17:01,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b2ae85ed58154b4d92aa2de511e635cd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b2ae85ed58154b4d92aa2de511e635cd 2024-12-02T17:17:01,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/8e6d19aed7b44ddd88d354ae0999e106 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/8e6d19aed7b44ddd88d354ae0999e106 2024-12-02T17:17:01,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a5cd5de0c2e74524b3f1216684245c29 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a5cd5de0c2e74524b3f1216684245c29 2024-12-02T17:17:01,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/96f570d195184c72a17fe4f9501eb25b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/96f570d195184c72a17fe4f9501eb25b 2024-12-02T17:17:01,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/61e099e4f16b41de95e5b397d5e97950 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/61e099e4f16b41de95e5b397d5e97950 2024-12-02T17:17:01,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/856d673c75de4e2f85bc5fc2d5e1474d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/856d673c75de4e2f85bc5fc2d5e1474d 2024-12-02T17:17:01,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d289005f407d422da1c5f902ec879469 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d289005f407d422da1c5f902ec879469 2024-12-02T17:17:01,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/ce37fcec31a940dbb1016e1b84819afa to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/ce37fcec31a940dbb1016e1b84819afa 2024-12-02T17:17:01,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/422b496b9f344c24bdee3e31a959b5dc to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/422b496b9f344c24bdee3e31a959b5dc 2024-12-02T17:17:01,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b7801489295745e7890080d8b2cfe88d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b7801489295745e7890080d8b2cfe88d 2024-12-02T17:17:01,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5694fbea9edc4134b5d7f688149b5d93 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5694fbea9edc4134b5d7f688149b5d93 2024-12-02T17:17:01,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/75e1c5697af94610928675a68cc7bbf5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/75e1c5697af94610928675a68cc7bbf5 2024-12-02T17:17:01,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d553ad152e6647a1beae8ec757f8f402 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d553ad152e6647a1beae8ec757f8f402 2024-12-02T17:17:01,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d419393db2174f18bb1d6e74cbf7df08 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/d419393db2174f18bb1d6e74cbf7df08 2024-12-02T17:17:01,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e1bc2fc15f7e403c813e540237b5eee2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e1bc2fc15f7e403c813e540237b5eee2 2024-12-02T17:17:01,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b3347e1978754b4990ff87c286c7633f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/b3347e1978754b4990ff87c286c7633f 2024-12-02T17:17:01,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/79144efab3454d64a3eb1a15afbfc43c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/79144efab3454d64a3eb1a15afbfc43c 2024-12-02T17:17:01,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/99c08b7b96fe4745a63f13f84e732e64 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/99c08b7b96fe4745a63f13f84e732e64 2024-12-02T17:17:01,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/49efa7da6b6648c5a494d849454cf0e1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/49efa7da6b6648c5a494d849454cf0e1 2024-12-02T17:17:01,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/0970cbf8030347a5ad2e650f6efe7a89 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/0970cbf8030347a5ad2e650f6efe7a89 2024-12-02T17:17:01,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e22832b64cb3452da600f519b669a25f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/e22832b64cb3452da600f519b669a25f 2024-12-02T17:17:01,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/48232d9f184f4495af014c5059fcb55e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/48232d9f184f4495af014c5059fcb55e 2024-12-02T17:17:01,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/81e6fd77672a474181a6c84bab4717f8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/81e6fd77672a474181a6c84bab4717f8 2024-12-02T17:17:01,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/099832a26b8a482b94c1a71aef7808d0 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/099832a26b8a482b94c1a71aef7808d0 2024-12-02T17:17:01,272 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/recovered.edits/491.seqid, newMaxSeqId=491, maxSeqId=1 2024-12-02T17:17:01,272 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. 2024-12-02T17:17:01,273 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for ccd4d5d4930a7d9225c7192b0068a2e3: Waiting for close lock at 1733159819951Running coprocessor pre-close hooks at 1733159819951Disabling compacts and flushes for region at 1733159819951Disabling writes for close at 1733159819951Obtaining lock to block concurrent updates at 1733159819951Preparing flush snapshotting stores in ccd4d5d4930a7d9225c7192b0068a2e3 at 1733159819951Finished memstore snapshotting TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733159819951Flushing stores of TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3. at 1733159819952 (+1 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/A: creating writer at 1733159819952Flushing ccd4d5d4930a7d9225c7192b0068a2e3/A: appending metadata at 1733159819954 (+2 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/A: closing flushed file at 1733159819954Flushing ccd4d5d4930a7d9225c7192b0068a2e3/B: creating writer at 1733159820363 (+409 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/B: appending metadata at 1733159820364 (+1 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/B: closing flushed file at 1733159820365 (+1 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/C: creating writer at 1733159820772 (+407 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/C: appending metadata at 1733159820773 (+1 ms)Flushing ccd4d5d4930a7d9225c7192b0068a2e3/C: closing flushed file at 1733159820773Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@337b5cac: reopening flushed file at 1733159821180 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29ddbaf5: reopening flushed file at 1733159821184 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c242707: reopening flushed file at 1733159821188 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ccd4d5d4930a7d9225c7192b0068a2e3 in 1241ms, sequenceid=488, compaction requested=true at 1733159821192 (+4 ms)Writing region close event to WAL at 1733159821269 (+77 ms)Running coprocessor post-close hooks at 1733159821272 (+3 ms)Closed at 1733159821272 2024-12-02T17:17:01,274 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:17:01,275 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=ccd4d5d4930a7d9225c7192b0068a2e3, regionState=CLOSED 2024-12-02T17:17:01,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:17:01,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-02T17:17:01,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure ccd4d5d4930a7d9225c7192b0068a2e3, server=3b3375d1a925,40723,1733159666211 in 1.4790 sec 2024-12-02T17:17:01,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-02T17:17:01,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ccd4d5d4930a7d9225c7192b0068a2e3, UNASSIGN in 1.4830 sec 2024-12-02T17:17:01,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T17:17:01,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4860 sec 2024-12-02T17:17:01,283 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159821282"}]},"ts":"1733159821282"} 2024-12-02T17:17:01,284 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T17:17:01,284 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T17:17:01,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4940 sec 2024-12-02T17:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T17:17:01,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:01,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T17:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:01,940 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T17:17:01,940 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:01,942 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:17:01,944 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/recovered.edits] 2024-12-02T17:17:01,946 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/12cb2d86dcc84fbb90df2914993577b5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/12cb2d86dcc84fbb90df2914993577b5 2024-12-02T17:17:01,947 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/17c19a1c66414b8782499f0fa8ad19c4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/17c19a1c66414b8782499f0fa8ad19c4 2024-12-02T17:17:01,948 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/8a4e4393449a448ba2eec6bc43855f11 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/8a4e4393449a448ba2eec6bc43855f11 2024-12-02T17:17:01,949 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/e14473ea6b234b6c8c9c9b0e68192074 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/A/e14473ea6b234b6c8c9c9b0e68192074 2024-12-02T17:17:01,950 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/59239a09559a463297dd79bc7a68014c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/59239a09559a463297dd79bc7a68014c 2024-12-02T17:17:01,951 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/99031365bc20436aa086193b00a960d7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/99031365bc20436aa086193b00a960d7 2024-12-02T17:17:01,952 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/b3ccb2418c7c41d3846fde8070566051 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/b3ccb2418c7c41d3846fde8070566051 2024-12-02T17:17:01,953 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/fe6ff23ef65e4eaabe54b468d1520855 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/B/fe6ff23ef65e4eaabe54b468d1520855 2024-12-02T17:17:01,955 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/36ab71efea5940c6be105f806d876eac to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/36ab71efea5940c6be105f806d876eac 2024-12-02T17:17:01,956 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5ed116df3f3a4779b910e80552b4220b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/5ed116df3f3a4779b910e80552b4220b 2024-12-02T17:17:01,957 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/9249b77c393f4236831ed040048c3075 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/9249b77c393f4236831ed040048c3075 2024-12-02T17:17:01,958 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a2d7abfb9d8f4a7bb017ef96b000fef6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/C/a2d7abfb9d8f4a7bb017ef96b000fef6 2024-12-02T17:17:01,960 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/recovered.edits/491.seqid to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3/recovered.edits/491.seqid 2024-12-02T17:17:01,960 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/ccd4d5d4930a7d9225c7192b0068a2e3 2024-12-02T17:17:01,960 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T17:17:01,962 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:01,964 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T17:17:01,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T17:17:01,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:01,966 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T17:17:01,967 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733159821966"}]},"ts":"9223372036854775807"} 2024-12-02T17:17:01,968 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T17:17:01,968 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ccd4d5d4930a7d9225c7192b0068a2e3, NAME => 'TestAcidGuarantees,,1733159792700.ccd4d5d4930a7d9225c7192b0068a2e3.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T17:17:01,968 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T17:17:01,968 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733159821968"}]},"ts":"9223372036854775807"} 2024-12-02T17:17:01,970 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T17:17:01,970 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:01,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-02T17:17:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T17:17:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T17:17:02,047 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:02,057 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 253), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1067 (was 1014) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1704 (was 2483) 2024-12-02T17:17:02,065 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1067, ProcessCount=11, AvailableMemoryMB=1704 2024-12-02T17:17:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:17:02,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:17:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:02,068 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T17:17:02,068 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:02,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-02T17:17:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-02T17:17:02,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T17:17:02,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742445_1621 (size=1061) 2024-12-02T17:17:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-02T17:17:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-02T17:17:02,475 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0 2024-12-02T17:17:02,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742446_1622 (size=53) 2024-12-02T17:17:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-02T17:17:02,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:17:02,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 305ba6d9a38ffb55509e7e10e4a765ff, disabling compactions & flushes 2024-12-02T17:17:02,880 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:02,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:02,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. after waiting 0 ms 2024-12-02T17:17:02,880 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:02,881 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:02,881 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 305ba6d9a38ffb55509e7e10e4a765ff: Waiting for close lock at 1733159822880Disabling compacts and flushes for region at 1733159822880Disabling writes for close at 1733159822880Writing region close event to WAL at 1733159822880Closed at 1733159822880 2024-12-02T17:17:02,883 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T17:17:02,883 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733159822883"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733159822883"}]},"ts":"1733159822883"} 2024-12-02T17:17:02,885 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T17:17:02,886 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T17:17:02,886 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159822886"}]},"ts":"1733159822886"} 2024-12-02T17:17:02,887 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-02T17:17:02,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, ASSIGN}] 2024-12-02T17:17:02,888 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, ASSIGN 2024-12-02T17:17:02,890 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, ASSIGN; state=OFFLINE, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=false 2024-12-02T17:17:03,040 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:03,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, ASSIGN because future has completed 2024-12-02T17:17:03,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:17:03,197 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:03,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:17:03,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:17:03,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,197 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,198 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,199 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:17:03,200 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ba6d9a38ffb55509e7e10e4a765ff columnFamilyName A 2024-12-02T17:17:03,200 DEBUG [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:03,200 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(327): Store=305ba6d9a38ffb55509e7e10e4a765ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:17:03,200 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,201 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:17:03,201 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ba6d9a38ffb55509e7e10e4a765ff columnFamilyName B 2024-12-02T17:17:03,201 DEBUG [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:03,201 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(327): Store=305ba6d9a38ffb55509e7e10e4a765ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:17:03,201 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,202 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:17:03,202 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ba6d9a38ffb55509e7e10e4a765ff columnFamilyName C 2024-12-02T17:17:03,202 DEBUG [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:03,202 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(327): Store=305ba6d9a38ffb55509e7e10e4a765ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:17:03,203 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:03,203 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,203 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,203 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,204 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,204 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,204 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:17:03,205 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,207 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T17:17:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-02T17:17:03,207 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 305ba6d9a38ffb55509e7e10e4a765ff; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60469868, jitterRate=-0.09892874956130981}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:17:03,207 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:03,208 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 305ba6d9a38ffb55509e7e10e4a765ff: Running coprocessor pre-open hook at 1733159823197Writing region info on filesystem at 1733159823198 (+1 ms)Initializing all the Stores at 1733159823198Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159823198Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159823198Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159823198Cleaning up temporary data from old regions at 1733159823204 (+6 ms)Running coprocessor post-open hooks at 1733159823207 (+3 ms)Region opened successfully at 1733159823208 (+1 ms) 2024-12-02T17:17:03,208 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., pid=146, masterSystemTime=1733159823194 2024-12-02T17:17:03,210 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:03,210 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:03,210 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=OPEN, openSeqNum=2, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:03,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:17:03,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-02T17:17:03,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 in 170 msec 2024-12-02T17:17:03,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-02T17:17:03,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, ASSIGN in 326 msec 2024-12-02T17:17:03,215 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T17:17:03,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159823215"}]},"ts":"1733159823215"} 2024-12-02T17:17:03,216 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-02T17:17:03,216 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T17:17:03,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-12-02T17:17:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-02T17:17:04,227 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:04,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:04,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:04,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:04,229 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4149e27e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:04,230 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:04,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc8a291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:04,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:04,232 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:04,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:04,233 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T17:17:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3b3375d1a925,41221,1733159665139 2024-12-02T17:17:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@21b25f26 2024-12-02T17:17:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T17:17:04,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T17:17:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-02T17:17:04,236 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T17:17:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:04,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742447_1623 (size=1097) 2024-12-02T17:17:04,646 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-02T17:17:04,646 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-02T17:17:04,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:17:04,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, REOPEN/MOVE}] 2024-12-02T17:17:04,651 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, REOPEN/MOVE 2024-12-02T17:17:04,651 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:04,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, REOPEN/MOVE because future has completed 2024-12-02T17:17:04,652 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:17:04,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:17:04,805 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:04,805 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:17:04,805 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 305ba6d9a38ffb55509e7e10e4a765ff, disabling compactions & flushes 2024-12-02T17:17:04,805 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:04,805 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:04,805 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. after waiting 0 ms 2024-12-02T17:17:04,805 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:04,808 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T17:17:04,808 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:04,808 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 305ba6d9a38ffb55509e7e10e4a765ff: Waiting for close lock at 1733159824805Running coprocessor pre-close hooks at 1733159824805Disabling compacts and flushes for region at 1733159824805Disabling writes for close at 1733159824805Writing region close event to WAL at 1733159824805Running coprocessor post-close hooks at 1733159824808 (+3 ms)Closed at 1733159824808 2024-12-02T17:17:04,808 WARN [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 305ba6d9a38ffb55509e7e10e4a765ff to self. 2024-12-02T17:17:04,810 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:04,810 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=CLOSED 2024-12-02T17:17:04,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:17:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-02T17:17:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 in 159 msec 2024-12-02T17:17:04,814 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, REOPEN/MOVE; state=CLOSED, location=3b3375d1a925,40723,1733159666211; forceNewPlan=false, retain=true 2024-12-02T17:17:04,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=OPENING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:04,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, REOPEN/MOVE because future has completed 2024-12-02T17:17:04,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:17:05,120 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,121 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} 2024-12-02T17:17:05,121 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,121 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T17:17:05,121 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,121 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,122 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,123 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:17:05,123 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ba6d9a38ffb55509e7e10e4a765ff columnFamilyName A 2024-12-02T17:17:05,124 DEBUG [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:05,124 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(327): Store=305ba6d9a38ffb55509e7e10e4a765ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:17:05,124 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,125 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:17:05,125 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ba6d9a38ffb55509e7e10e4a765ff columnFamilyName B 2024-12-02T17:17:05,125 DEBUG [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:05,125 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(327): Store=305ba6d9a38ffb55509e7e10e4a765ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:17:05,125 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,126 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-02T17:17:05,126 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 305ba6d9a38ffb55509e7e10e4a765ff columnFamilyName C 2024-12-02T17:17:05,126 DEBUG [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:05,126 INFO [StoreOpener-305ba6d9a38ffb55509e7e10e4a765ff-1 {}] regionserver.HStore(327): Store=305ba6d9a38ffb55509e7e10e4a765ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T17:17:05,127 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,127 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,127 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,128 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,129 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,129 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,129 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T17:17:05,130 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,131 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 305ba6d9a38ffb55509e7e10e4a765ff; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66077720, jitterRate=-0.015365242958068848}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T17:17:05,131 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,131 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 305ba6d9a38ffb55509e7e10e4a765ff: Running coprocessor pre-open hook at 1733159825121Writing region info on filesystem at 1733159825121Initializing all the Stores at 1733159825122 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159825122Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159825122Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733159825122Cleaning up temporary data from old regions at 1733159825129 (+7 ms)Running coprocessor post-open hooks at 1733159825131 (+2 ms)Region opened successfully at 1733159825131 2024-12-02T17:17:05,132 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., pid=151, masterSystemTime=1733159825118 2024-12-02T17:17:05,133 DEBUG [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,133 INFO [RS_OPEN_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,134 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=OPEN, openSeqNum=5, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:17:05,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-02T17:17:05,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 in 169 msec 2024-12-02T17:17:05,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-02T17:17:05,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, REOPEN/MOVE in 487 msec 2024-12-02T17:17:05,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-02T17:17:05,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-12-02T17:17:05,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-12-02T17:17:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-02T17:17:05,142 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:05,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,144 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7912f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,145 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5f56dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,147 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,147 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32884bd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,148 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e0c8f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,149 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,150 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fea2668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6f4ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,153 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fee296a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,154 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5698bfed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,155 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d3349e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242ed84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c10d4d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35dac407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,160 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5582c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44684b40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d37f98f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,165 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b0b207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2517816d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@96fd646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd2bfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bc56f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3b3375d1a925,41221,-1 for getting cluster id 2024-12-02T17:17:05,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T17:17:05,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd8a279c-f176-4ff6-92c2-4858730f937a' 2024-12-02T17:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T17:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd8a279c-f176-4ff6-92c2-4858730f937a" 2024-12-02T17:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5a9a50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3b3375d1a925,41221,-1] 2024-12-02T17:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T17:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:05,170 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T17:17:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a88e1fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T17:17:05,171 DEBUG [Thread-2739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,171 DEBUG [Thread-2750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,171 DEBUG [Thread-2741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,171 DEBUG [Thread-2756 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:05,172 DEBUG [Thread-2758 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,172 DEBUG [Thread-2754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-02T17:17:05,172 DEBUG [Thread-2743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T17:17:05,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,173 DEBUG [Thread-2745 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,174 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:05,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:05,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,174 DEBUG [Thread-2752 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,176 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,180 DEBUG [Thread-2747 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T17:17:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,181 DEBUG [Thread-2747 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3b3375d1a925,40723,1733159666211, seqNum=-1] 2024-12-02T17:17:05,181 DEBUG [Thread-2747 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T17:17:05,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T17:17:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5] 2024-12-02T17:17:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-02T17:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:05,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:05,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159825198, max timestamp=1733159825209Num uniques 859; 2024-12-02T17:17:05,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:17:05,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159825198, max timestamp=1733159825209Num uniques 859; 2024-12-02T17:17:05,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733159825198, max timestamp=1733159825209Num uniques 859; 2024-12-02T17:17:05,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:17:05,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-02T17:17:05,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028ede4f08a38a403190f6ded4cb384efa_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159825185/Put/seqid=0 2024-12-02T17:17:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742448_1624 (size=12154) 2024-12-02T17:17:05,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159885259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159885265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159885265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159885266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159885268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T17:17:05,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159885377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159885378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159885378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159885378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159885378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T17:17:05,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159885587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159885588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159885588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159885588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159885588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:05,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028ede4f08a38a403190f6ded4cb384efa_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028ede4f08a38a403190f6ded4cb384efa_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:05,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ead07f93900e4895a94b94ab1425f429, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:05,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ead07f93900e4895a94b94ab1425f429 is 175, key is test_row_0/A:col10/1733159825185/Put/seqid=0 2024-12-02T17:17:05,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742449_1625 (size=30955) 2024-12-02T17:17:05,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ead07f93900e4895a94b94ab1425f429 2024-12-02T17:17:05,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/77d2c25416934eef91e9ef5087a6f9e7 is 50, key is test_row_0/B:col10/1733159825185/Put/seqid=0 2024-12-02T17:17:05,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742450_1626 (size=12001) 2024-12-02T17:17:05,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T17:17:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159885897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159885897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159885897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159885898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159885898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:05,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:05,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:06,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:06,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:06,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:06,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/77d2c25416934eef91e9ef5087a6f9e7 2024-12-02T17:17:06,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/6709e62bc52e4221b3330a93824c7212 is 50, key is test_row_0/C:col10/1733159825185/Put/seqid=0 2024-12-02T17:17:06,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742451_1627 (size=12001) 2024-12-02T17:17:06,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/6709e62bc52e4221b3330a93824c7212 2024-12-02T17:17:06,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ead07f93900e4895a94b94ab1425f429 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429 2024-12-02T17:17:06,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429, entries=150, sequenceid=15, filesize=30.2 K 2024-12-02T17:17:06,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/77d2c25416934eef91e9ef5087a6f9e7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/77d2c25416934eef91e9ef5087a6f9e7 2024-12-02T17:17:06,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/77d2c25416934eef91e9ef5087a6f9e7, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T17:17:06,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/6709e62bc52e4221b3330a93824c7212 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/6709e62bc52e4221b3330a93824c7212 2024-12-02T17:17:06,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/6709e62bc52e4221b3330a93824c7212, entries=150, sequenceid=15, filesize=11.7 K 2024-12-02T17:17:06,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 305ba6d9a38ffb55509e7e10e4a765ff in 1049ms, sequenceid=15, compaction requested=false 2024-12-02T17:17:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-02T17:17:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:06,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-02T17:17:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:06,258 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-02T17:17:06,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:06,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028561825beac9456e96d323acc5c1601d_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159825264/Put/seqid=0 2024-12-02T17:17:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T17:17:06,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742452_1628 (size=12154) 2024-12-02T17:17:06,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:06,338 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028561825beac9456e96d323acc5c1601d_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028561825beac9456e96d323acc5c1601d_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/566f9a20cedc467fbbd66a43947f8a60, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/566f9a20cedc467fbbd66a43947f8a60 is 175, key is test_row_0/A:col10/1733159825264/Put/seqid=0 2024-12-02T17:17:06,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742453_1629 (size=30955) 2024-12-02T17:17:06,347 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/566f9a20cedc467fbbd66a43947f8a60 2024-12-02T17:17:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/15d2d08a358644c0aa61a8ec280062f2 is 50, key is test_row_0/B:col10/1733159825264/Put/seqid=0 2024-12-02T17:17:06,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742454_1630 (size=12001) 2024-12-02T17:17:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:06,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159886416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159886417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159886420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159886422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159886422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159886527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159886527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159886527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159886536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159886537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159886737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159886737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159886737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159886747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159886747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:06,766 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/15d2d08a358644c0aa61a8ec280062f2 2024-12-02T17:17:06,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/cdffde5a32674de7b11d048e923365e4 is 50, key is test_row_0/C:col10/1733159825264/Put/seqid=0 2024-12-02T17:17:06,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742455_1631 (size=12001) 2024-12-02T17:17:06,810 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T17:17:07,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159887047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159887047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159887048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159887057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159887057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,178 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/cdffde5a32674de7b11d048e923365e4 2024-12-02T17:17:07,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/566f9a20cedc467fbbd66a43947f8a60 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60 2024-12-02T17:17:07,188 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60, entries=150, sequenceid=49, filesize=30.2 K 2024-12-02T17:17:07,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/15d2d08a358644c0aa61a8ec280062f2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/15d2d08a358644c0aa61a8ec280062f2 2024-12-02T17:17:07,192 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/15d2d08a358644c0aa61a8ec280062f2, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T17:17:07,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/cdffde5a32674de7b11d048e923365e4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/cdffde5a32674de7b11d048e923365e4 2024-12-02T17:17:07,196 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/cdffde5a32674de7b11d048e923365e4, entries=150, sequenceid=49, filesize=11.7 K 2024-12-02T17:17:07,197 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 305ba6d9a38ffb55509e7e10e4a765ff in 938ms, sequenceid=49, compaction requested=false 2024-12-02T17:17:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:07,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-02T17:17:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-02T17:17:07,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-02T17:17:07,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0240 sec 2024-12-02T17:17:07,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0300 sec 2024-12-02T17:17:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-02T17:17:07,317 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:07,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-02T17:17:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T17:17:07,320 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:07,321 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:07,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T17:17:07,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-02T17:17:07,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:07,474 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:17:07,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c7f62f3695824ec38b211ec1908d0cdd_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159826418/Put/seqid=0 2024-12-02T17:17:07,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742456_1632 (size=12154) 2024-12-02T17:17:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:07,490 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c7f62f3695824ec38b211ec1908d0cdd_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7f62f3695824ec38b211ec1908d0cdd_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:07,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f1c5ed27397c4c139588bc482076c484, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:07,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f1c5ed27397c4c139588bc482076c484 is 175, key is test_row_0/A:col10/1733159826418/Put/seqid=0 2024-12-02T17:17:07,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742457_1633 (size=30955) 2024-12-02T17:17:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:07,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:07,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159887580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159887581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159887581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159887582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159887582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T17:17:07,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159887697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159887908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159887908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159887908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159887908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159887908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:07,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:07,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:07,916 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f1c5ed27397c4c139588bc482076c484 2024-12-02T17:17:07,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/00e0973846d74b08a2988e1febd79039 is 50, key is test_row_0/B:col10/1733159826418/Put/seqid=0 2024-12-02T17:17:07,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742458_1634 (size=12001) 2024-12-02T17:17:07,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T17:17:08,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159888216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159888216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159888216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159888217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159888217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,327 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/00e0973846d74b08a2988e1febd79039 2024-12-02T17:17:08,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/447e39371b34454b9784194e52559f02 is 50, key is test_row_0/C:col10/1733159826418/Put/seqid=0 2024-12-02T17:17:08,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742459_1635 (size=12001) 2024-12-02T17:17:08,338 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/447e39371b34454b9784194e52559f02 2024-12-02T17:17:08,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f1c5ed27397c4c139588bc482076c484 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484 2024-12-02T17:17:08,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484, entries=150, sequenceid=61, filesize=30.2 K 2024-12-02T17:17:08,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/00e0973846d74b08a2988e1febd79039 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/00e0973846d74b08a2988e1febd79039 2024-12-02T17:17:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,351 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/00e0973846d74b08a2988e1febd79039, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T17:17:08,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/447e39371b34454b9784194e52559f02 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/447e39371b34454b9784194e52559f02 2024-12-02T17:17:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,355 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/447e39371b34454b9784194e52559f02, entries=150, sequenceid=61, filesize=11.7 K 2024-12-02T17:17:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,356 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 305ba6d9a38ffb55509e7e10e4a765ff in 882ms, sequenceid=61, compaction requested=true 2024-12-02T17:17:08,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:08,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-02T17:17:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-02T17:17:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-02T17:17:08,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0410 sec 2024-12-02T17:17:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0460 sec 2024-12-02T17:17:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-02T17:17:08,458 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-02T17:17:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T17:17:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,463 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,467 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:08,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T17:17:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-02T17:17:08,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:08,625 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:17:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029ab3384bc6ae4c03b0d0feaf147200da_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159827581/Put/seqid=0 2024-12-02T17:17:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742460_1636 (size=9714) 2024-12-02T17:17:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159888749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159888750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159888752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159888752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159888754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T17:17:08,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159888857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159888867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159888867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159888877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159888877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159889067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159889076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159889077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T17:17:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159889087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159889087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:09,107 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412029ab3384bc6ae4c03b0d0feaf147200da_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ab3384bc6ae4c03b0d0feaf147200da_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:09,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/c40ad5d87ec94c21ac692f744e719a07, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/c40ad5d87ec94c21ac692f744e719a07 is 175, key is test_row_0/A:col10/1733159827581/Put/seqid=0 2024-12-02T17:17:09,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742461_1637 (size=22361) 2024-12-02T17:17:09,113 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/c40ad5d87ec94c21ac692f744e719a07 2024-12-02T17:17:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e0ff749f75574eff8aa54c5e86e22336 is 50, key is test_row_0/B:col10/1733159827581/Put/seqid=0 2024-12-02T17:17:09,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742462_1638 (size=9657) 2024-12-02T17:17:09,127 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e0ff749f75574eff8aa54c5e86e22336 2024-12-02T17:17:09,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/db476286b77540b8a32263046f8a0bb6 is 50, key is test_row_0/C:col10/1733159827581/Put/seqid=0 2024-12-02T17:17:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742463_1639 (size=9657) 2024-12-02T17:17:09,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/db476286b77540b8a32263046f8a0bb6 2024-12-02T17:17:09,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/c40ad5d87ec94c21ac692f744e719a07 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07 2024-12-02T17:17:09,147 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07, entries=100, sequenceid=81, filesize=21.8 K 2024-12-02T17:17:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e0ff749f75574eff8aa54c5e86e22336 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e0ff749f75574eff8aa54c5e86e22336 2024-12-02T17:17:09,167 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e0ff749f75574eff8aa54c5e86e22336, entries=100, sequenceid=81, filesize=9.4 K 2024-12-02T17:17:09,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/db476286b77540b8a32263046f8a0bb6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/db476286b77540b8a32263046f8a0bb6 2024-12-02T17:17:09,179 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/db476286b77540b8a32263046f8a0bb6, entries=100, sequenceid=81, filesize=9.4 K 2024-12-02T17:17:09,181 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 305ba6d9a38ffb55509e7e10e4a765ff in 555ms, sequenceid=81, compaction requested=true 2024-12-02T17:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:09,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-02T17:17:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-02T17:17:09,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-02T17:17:09,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 715 msec 2024-12-02T17:17:09,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 730 msec 2024-12-02T17:17:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:09,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:17:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:09,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:09,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202195ef9455d4f4540a55ab1cd6f63005e_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:09,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742464_1640 (size=14594) 2024-12-02T17:17:09,392 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:09,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202195ef9455d4f4540a55ab1cd6f63005e_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202195ef9455d4f4540a55ab1cd6f63005e_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:09,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/aded406dc1dd44608022431ce40cd2be, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:09,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/aded406dc1dd44608022431ce40cd2be is 175, key is test_row_0/A:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:09,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742465_1641 (size=39549) 2024-12-02T17:17:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159889416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159889417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159889420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159889423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/aded406dc1dd44608022431ce40cd2be 2024-12-02T17:17:09,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159889426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/abfa07c4e4ee453189f0a0008100e0ab is 50, key is test_row_0/B:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:09,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742466_1642 (size=12001) 2024-12-02T17:17:09,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159889537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159889537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159889537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159889537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159889538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T17:17:09,597 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:09,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-02T17:17:09,612 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:09,613 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:09,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:09,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159889747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159889747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159889747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159889748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159889757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:09,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:09,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:09,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:09,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:09,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/abfa07c4e4ee453189f0a0008100e0ab 2024-12-02T17:17:09,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/11d65a1085af45269b05ab4fbe5669fd is 50, key is test_row_0/C:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:09,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:09,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:09,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742467_1643 (size=12001) 2024-12-02T17:17:09,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/11d65a1085af45269b05ab4fbe5669fd 2024-12-02T17:17:09,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/aded406dc1dd44608022431ce40cd2be as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be 2024-12-02T17:17:10,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be, entries=200, sequenceid=95, filesize=38.6 K 2024-12-02T17:17:10,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/abfa07c4e4ee453189f0a0008100e0ab as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/abfa07c4e4ee453189f0a0008100e0ab 2024-12-02T17:17:10,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/abfa07c4e4ee453189f0a0008100e0ab, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T17:17:10,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/11d65a1085af45269b05ab4fbe5669fd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/11d65a1085af45269b05ab4fbe5669fd 2024-12-02T17:17:10,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/11d65a1085af45269b05ab4fbe5669fd, entries=150, sequenceid=95, filesize=11.7 K 2024-12-02T17:17:10,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 305ba6d9a38ffb55509e7e10e4a765ff in 644ms, sequenceid=95, compaction requested=true 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:10,025 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:10,026 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T17:17:10,038 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 154775 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T17:17:10,039 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:10,039 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,039 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=151.1 K 2024-12-02T17:17:10,039 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,039 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be] 2024-12-02T17:17:10,041 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T17:17:10,042 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:10,042 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,042 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/77d2c25416934eef91e9ef5087a6f9e7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/15d2d08a358644c0aa61a8ec280062f2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/00e0973846d74b08a2988e1febd79039, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e0ff749f75574eff8aa54c5e86e22336, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/abfa07c4e4ee453189f0a0008100e0ab] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=56.3 K 2024-12-02T17:17:10,043 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ead07f93900e4895a94b94ab1425f429, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733159825180 2024-12-02T17:17:10,046 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d2c25416934eef91e9ef5087a6f9e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733159825180 2024-12-02T17:17:10,046 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d2d08a358644c0aa61a8ec280062f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733159825262 2024-12-02T17:17:10,046 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 566f9a20cedc467fbbd66a43947f8a60, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733159825262 2024-12-02T17:17:10,046 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e0973846d74b08a2988e1febd79039, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733159826418 2024-12-02T17:17:10,047 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c5ed27397c4c139588bc482076c484, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733159826418 2024-12-02T17:17:10,047 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0ff749f75574eff8aa54c5e86e22336, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733159827571 2024-12-02T17:17:10,049 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c40ad5d87ec94c21ac692f744e719a07, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733159827571 2024-12-02T17:17:10,052 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting abfa07c4e4ee453189f0a0008100e0ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159828747 2024-12-02T17:17:10,056 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting aded406dc1dd44608022431ce40cd2be, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159828747 2024-12-02T17:17:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:10,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:10,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:10,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:10,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,100 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#552 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:10,101 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/bcd32e2f262442e7a62baf6d8ee58616 is 50, key is test_row_0/B:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:10,110 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:10,120 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412026fca31e0b3b24874845392cb5a4740df_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:10,123 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412026fca31e0b3b24874845392cb5a4740df_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:10,123 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026fca31e0b3b24874845392cb5a4740df_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:10,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202917bd5e8e2c44ea8808436ae53350bcd_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159829413/Put/seqid=0 2024-12-02T17:17:10,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159890113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159890117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159890117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742468_1644 (size=12173) 2024-12-02T17:17:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159890146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159890147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742469_1645 (size=4469) 2024-12-02T17:17:10,171 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/bcd32e2f262442e7a62baf6d8ee58616 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/bcd32e2f262442e7a62baf6d8ee58616 2024-12-02T17:17:10,172 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#553 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:10,172 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ac9521f807044925a66f80a2d28773d7 is 175, key is test_row_0/A:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:10,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742470_1646 (size=12154) 2024-12-02T17:17:10,193 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:10,201 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into bcd32e2f262442e7a62baf6d8ee58616(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:10,201 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:10,201 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=11, startTime=1733159830025; duration=0sec 2024-12-02T17:17:10,201 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:10,201 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:10,201 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-02T17:17:10,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202917bd5e8e2c44ea8808436ae53350bcd_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202917bd5e8e2c44ea8808436ae53350bcd_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:10,221 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/1cb1caf82e5a4d599a0a7bb5f8168995, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:10,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/1cb1caf82e5a4d599a0a7bb5f8168995 is 175, key is test_row_0/A:col10/1733159829413/Put/seqid=0 2024-12-02T17:17:10,222 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-02T17:17:10,222 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:10,222 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,222 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/6709e62bc52e4221b3330a93824c7212, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/cdffde5a32674de7b11d048e923365e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/447e39371b34454b9784194e52559f02, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/db476286b77540b8a32263046f8a0bb6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/11d65a1085af45269b05ab4fbe5669fd] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=56.3 K 2024-12-02T17:17:10,225 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6709e62bc52e4221b3330a93824c7212, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733159825180 2024-12-02T17:17:10,228 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdffde5a32674de7b11d048e923365e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733159825262 2024-12-02T17:17:10,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742471_1647 (size=31127) 2024-12-02T17:17:10,237 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447e39371b34454b9784194e52559f02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733159826418 2024-12-02T17:17:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:10,240 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting db476286b77540b8a32263046f8a0bb6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733159827571 2024-12-02T17:17:10,241 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d65a1085af45269b05ab4fbe5669fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159828747 2024-12-02T17:17:10,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159890248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159890249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159890249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742472_1648 (size=30955) 2024-12-02T17:17:10,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/1cb1caf82e5a4d599a0a7bb5f8168995 2024-12-02T17:17:10,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159890267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,280 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#555 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:10,280 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/df18c97a69184f9dbb91a539eedb9fbd is 50, key is test_row_0/C:col10/1733159828747/Put/seqid=0 2024-12-02T17:17:10,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/0888356376404c9ba1bd5dc4a683d689 is 50, key is test_row_0/B:col10/1733159829413/Put/seqid=0 2024-12-02T17:17:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159890268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742473_1649 (size=12173) 2024-12-02T17:17:10,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742474_1650 (size=12001) 2024-12-02T17:17:10,353 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/df18c97a69184f9dbb91a539eedb9fbd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/df18c97a69184f9dbb91a539eedb9fbd 2024-12-02T17:17:10,360 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into df18c97a69184f9dbb91a539eedb9fbd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:10,360 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:10,360 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=11, startTime=1733159830025; duration=0sec 2024-12-02T17:17:10,360 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:10,360 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:10,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:10,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:10,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159890461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159890467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159890468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159890487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159890487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:10,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,646 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ac9521f807044925a66f80a2d28773d7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ac9521f807044925a66f80a2d28773d7 2024-12-02T17:17:10,651 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into ac9521f807044925a66f80a2d28773d7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:10,651 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:10,651 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=11, startTime=1733159830025; duration=0sec 2024-12-02T17:17:10,651 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:10,651 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:10,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:10,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/0888356376404c9ba1bd5dc4a683d689 2024-12-02T17:17:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:10,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/ceb8a9837378461c82bfc66d957f7ef9 is 50, key is test_row_0/C:col10/1733159829413/Put/seqid=0 2024-12-02T17:17:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159890781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159890781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159890789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159890801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:10,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742475_1651 (size=12001) 2024-12-02T17:17:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159890801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:10,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/ceb8a9837378461c82bfc66d957f7ef9 2024-12-02T17:17:10,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/1cb1caf82e5a4d599a0a7bb5f8168995 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995 2024-12-02T17:17:10,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995, entries=150, sequenceid=115, filesize=30.2 K 2024-12-02T17:17:10,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/0888356376404c9ba1bd5dc4a683d689 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0888356376404c9ba1bd5dc4a683d689 2024-12-02T17:17:10,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0888356376404c9ba1bd5dc4a683d689, entries=150, sequenceid=115, filesize=11.7 K 2024-12-02T17:17:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/ceb8a9837378461c82bfc66d957f7ef9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/ceb8a9837378461c82bfc66d957f7ef9 2024-12-02T17:17:10,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/ceb8a9837378461c82bfc66d957f7ef9, entries=150, sequenceid=115, filesize=11.7 K 2024-12-02T17:17:10,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 305ba6d9a38ffb55509e7e10e4a765ff in 803ms, sequenceid=115, compaction requested=false 2024-12-02T17:17:10,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:10,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-02T17:17:10,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:10,888 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:10,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d06845d749b841deba5d76f65d7b3449_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159830112/Put/seqid=0 2024-12-02T17:17:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742476_1652 (size=12154) 2024-12-02T17:17:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:11,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:11,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159891329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159891333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159891337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159891339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159891345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:11,376 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202d06845d749b841deba5d76f65d7b3449_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d06845d749b841deba5d76f65d7b3449_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ee1e3e2042b84bfc93a725e56910487f, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:11,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ee1e3e2042b84bfc93a725e56910487f is 175, key is test_row_0/A:col10/1733159830112/Put/seqid=0 2024-12-02T17:17:11,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742477_1653 (size=30955) 2024-12-02T17:17:11,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159891451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159891452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,460 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159891459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159891459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159891459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159891667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159891668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159891677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159891677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159891677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:11,827 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ee1e3e2042b84bfc93a725e56910487f 2024-12-02T17:17:11,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/045ca978596e47bb9747dae68e03271f is 50, key is test_row_0/B:col10/1733159830112/Put/seqid=0 2024-12-02T17:17:11,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742478_1654 (size=12001) 2024-12-02T17:17:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159891977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159891987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159891987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159891987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:11,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:11,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159891987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:11,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:12,265 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/045ca978596e47bb9747dae68e03271f 2024-12-02T17:17:12,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/0d35e281bc2f442ea2fe8b6cafab4b43 is 50, key is test_row_0/C:col10/1733159830112/Put/seqid=0 2024-12-02T17:17:12,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742479_1655 (size=12001) 2024-12-02T17:17:12,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159892489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:12,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159892507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:12,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159892507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:12,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159892507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159892509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:12,691 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/0d35e281bc2f442ea2fe8b6cafab4b43 2024-12-02T17:17:12,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/ee1e3e2042b84bfc93a725e56910487f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f 2024-12-02T17:17:12,739 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f, entries=150, sequenceid=131, filesize=30.2 K 2024-12-02T17:17:12,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/045ca978596e47bb9747dae68e03271f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/045ca978596e47bb9747dae68e03271f 2024-12-02T17:17:12,752 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/045ca978596e47bb9747dae68e03271f, entries=150, sequenceid=131, filesize=11.7 K 2024-12-02T17:17:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/0d35e281bc2f442ea2fe8b6cafab4b43 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0d35e281bc2f442ea2fe8b6cafab4b43 2024-12-02T17:17:12,775 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0d35e281bc2f442ea2fe8b6cafab4b43, entries=150, sequenceid=131, filesize=11.7 K 2024-12-02T17:17:12,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 305ba6d9a38ffb55509e7e10e4a765ff in 1890ms, sequenceid=131, compaction requested=true 2024-12-02T17:17:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-02T17:17:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-02T17:17:12,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-02T17:17:12,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1680 sec 2024-12-02T17:17:12,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.1810 sec 2024-12-02T17:17:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:13,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:17:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:13,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cbed7d5497ff47f4b31f4b57c221aa2f_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:13,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159893549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159893550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159893556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159893556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159893561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742480_1656 (size=14794) 2024-12-02T17:17:13,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:13,632 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cbed7d5497ff47f4b31f4b57c221aa2f_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cbed7d5497ff47f4b31f4b57c221aa2f_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:13,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f726c91bfe164a88bd58bc8ef537d336, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:13,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f726c91bfe164a88bd58bc8ef537d336 is 175, key is test_row_0/A:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159893667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159893671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159893672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159893678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742481_1657 (size=39749) 2024-12-02T17:17:13,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159893685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T17:17:13,788 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:13,789 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-02T17:17:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T17:17:13,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:13,812 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:13,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:13,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159893876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159893887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159893887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159893887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T17:17:13,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159893909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:13,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:13,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:13,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f726c91bfe164a88bd58bc8ef537d336 2024-12-02T17:17:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T17:17:14,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/0018f87381eb49b9a621474c2eafdae4 is 50, key is test_row_0/B:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159894188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159894197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159894207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159894207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742482_1658 (size=12151) 2024-12-02T17:17:14,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/0018f87381eb49b9a621474c2eafdae4 2024-12-02T17:17:14,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159894227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/870e72fc0fa74e99861df96dab6c8e10 is 50, key is test_row_0/C:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:14,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742483_1659 (size=12151) 2024-12-02T17:17:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T17:17:14,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:14,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:14,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159894698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159894706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159894718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159894727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/870e72fc0fa74e99861df96dab6c8e10 2024-12-02T17:17:14,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159894737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:14,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f726c91bfe164a88bd58bc8ef537d336 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336 2024-12-02T17:17:14,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:14,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:14,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:14,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336, entries=200, sequenceid=151, filesize=38.8 K 2024-12-02T17:17:14,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/0018f87381eb49b9a621474c2eafdae4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0018f87381eb49b9a621474c2eafdae4 2024-12-02T17:17:14,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0018f87381eb49b9a621474c2eafdae4, entries=150, sequenceid=151, filesize=11.9 K 2024-12-02T17:17:14,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/870e72fc0fa74e99861df96dab6c8e10 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/870e72fc0fa74e99861df96dab6c8e10 2024-12-02T17:17:14,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/870e72fc0fa74e99861df96dab6c8e10, entries=150, sequenceid=151, filesize=11.9 K 2024-12-02T17:17:14,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 305ba6d9a38ffb55509e7e10e4a765ff in 1378ms, sequenceid=151, compaction requested=true 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:14,891 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:14,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:14,891 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:17:14,910 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:17:14,910 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:14,910 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132786 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:17:14,910 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:14,910 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,910 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,910 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ac9521f807044925a66f80a2d28773d7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=129.7 K 2024-12-02T17:17:14,910 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/bcd32e2f262442e7a62baf6d8ee58616, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0888356376404c9ba1bd5dc4a683d689, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/045ca978596e47bb9747dae68e03271f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0018f87381eb49b9a621474c2eafdae4] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=47.2 K 2024-12-02T17:17:14,910 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,910 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ac9521f807044925a66f80a2d28773d7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336] 2024-12-02T17:17:14,913 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd32e2f262442e7a62baf6d8ee58616, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159828747 2024-12-02T17:17:14,916 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac9521f807044925a66f80a2d28773d7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159828747 2024-12-02T17:17:14,918 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0888356376404c9ba1bd5dc4a683d689, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733159829413 2024-12-02T17:17:14,920 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 045ca978596e47bb9747dae68e03271f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733159830103 2024-12-02T17:17:14,922 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb1caf82e5a4d599a0a7bb5f8168995, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733159829413 2024-12-02T17:17:14,932 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0018f87381eb49b9a621474c2eafdae4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733159831334 2024-12-02T17:17:14,932 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee1e3e2042b84bfc93a725e56910487f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733159830103 2024-12-02T17:17:14,935 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f726c91bfe164a88bd58bc8ef537d336, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733159831326 2024-12-02T17:17:14,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:14,936 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T17:17:14,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120277270236efdb46548dbdf3785e925334_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159833549/Put/seqid=0 2024-12-02T17:17:14,998 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#565 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:14,999 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/d8ff06f15860467a8040903b573bf244 is 50, key is test_row_0/B:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:15,001 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:15,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742484_1660 (size=12304) 2024-12-02T17:17:15,034 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202efb8a0bd409846fea8240bb680435dbf_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:15,036 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202efb8a0bd409846fea8240bb680435dbf_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:15,036 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202efb8a0bd409846fea8240bb680435dbf_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:15,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742485_1661 (size=12459) 2024-12-02T17:17:15,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742486_1662 (size=4469) 2024-12-02T17:17:15,078 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/d8ff06f15860467a8040903b573bf244 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d8ff06f15860467a8040903b573bf244 2024-12-02T17:17:15,112 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into d8ff06f15860467a8040903b573bf244(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:15,112 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:15,112 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=12, startTime=1733159834891; duration=0sec 2024-12-02T17:17:15,112 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:15,112 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:15,112 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:17:15,126 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:17:15,127 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:15,127 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:15,127 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/df18c97a69184f9dbb91a539eedb9fbd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/ceb8a9837378461c82bfc66d957f7ef9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0d35e281bc2f442ea2fe8b6cafab4b43, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/870e72fc0fa74e99861df96dab6c8e10] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=47.2 K 2024-12-02T17:17:15,128 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting df18c97a69184f9dbb91a539eedb9fbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733159828747 2024-12-02T17:17:15,129 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb8a9837378461c82bfc66d957f7ef9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733159829413 2024-12-02T17:17:15,129 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d35e281bc2f442ea2fe8b6cafab4b43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733159830103 2024-12-02T17:17:15,129 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 870e72fc0fa74e99861df96dab6c8e10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733159831334 2024-12-02T17:17:15,171 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#567 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:15,172 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/992269e8a62f4f6a9057bb3f30fda279 is 50, key is test_row_0/C:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:15,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742487_1663 (size=12459) 2024-12-02T17:17:15,213 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/992269e8a62f4f6a9057bb3f30fda279 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/992269e8a62f4f6a9057bb3f30fda279 2024-12-02T17:17:15,246 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into 992269e8a62f4f6a9057bb3f30fda279(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:15,246 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:15,246 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=12, startTime=1733159834891; duration=0sec 2024-12-02T17:17:15,246 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:15,246 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:15,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,444 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120277270236efdb46548dbdf3785e925334_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120277270236efdb46548dbdf3785e925334_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/743bd2bb46ce463eaa02a2f55a6dfbaa, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:15,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/743bd2bb46ce463eaa02a2f55a6dfbaa is 175, key is test_row_0/A:col10/1733159833549/Put/seqid=0 2024-12-02T17:17:15,481 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#566 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:15,482 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/d9bb87efd0c442ee8a3e592294c13120 is 175, key is test_row_0/A:col10/1733159831334/Put/seqid=0 2024-12-02T17:17:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742488_1664 (size=31105) 2024-12-02T17:17:15,501 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/743bd2bb46ce463eaa02a2f55a6dfbaa 2024-12-02T17:17:15,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742489_1665 (size=31413) 2024-12-02T17:17:15,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/1125cb6a8f2f4d9e817fce1e767f3300 is 50, key is test_row_0/B:col10/1733159833549/Put/seqid=0 2024-12-02T17:17:15,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742490_1666 (size=12151) 2024-12-02T17:17:15,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/1125cb6a8f2f4d9e817fce1e767f3300 2024-12-02T17:17:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/5035ecf7978c4b69974ce38b89c15087 is 50, key is test_row_0/C:col10/1733159833549/Put/seqid=0 2024-12-02T17:17:15,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742491_1667 (size=12151) 2024-12-02T17:17:15,691 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/5035ecf7978c4b69974ce38b89c15087 2024-12-02T17:17:15,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/743bd2bb46ce463eaa02a2f55a6dfbaa as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa 2024-12-02T17:17:15,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:15,754 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa, entries=150, sequenceid=163, filesize=30.4 K 2024-12-02T17:17:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/1125cb6a8f2f4d9e817fce1e767f3300 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/1125cb6a8f2f4d9e817fce1e767f3300 2024-12-02T17:17:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,782 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/1125cb6a8f2f4d9e817fce1e767f3300, entries=150, sequenceid=163, filesize=11.9 K 2024-12-02T17:17:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/5035ecf7978c4b69974ce38b89c15087 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/5035ecf7978c4b69974ce38b89c15087 2024-12-02T17:17:15,803 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/5035ecf7978c4b69974ce38b89c15087, entries=150, sequenceid=163, filesize=11.9 K 2024-12-02T17:17:15,848 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=93.93 KB/96180 for 305ba6d9a38ffb55509e7e10e4a765ff in 912ms, sequenceid=163, compaction requested=false 2024-12-02T17:17:15,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:15,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:15,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-02T17:17:15,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-02T17:17:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-02T17:17:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:15,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:15,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-02T17:17:15,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0450 sec 2024-12-02T17:17:15,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0740 sec 2024-12-02T17:17:15,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202261187f20b0144c39b6dfd3bb41a0863_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742492_1668 (size=12304) 2024-12-02T17:17:15,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159895887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:15,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:15,902 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:15,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159895891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:15,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202261187f20b0144c39b6dfd3bb41a0863_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202261187f20b0144c39b6dfd3bb41a0863_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:15,909 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/9a301e8c138d4922bdea5e4d4c8ce81f, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:15,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/9a301e8c138d4922bdea5e4d4c8ce81f is 175, key is test_row_0/A:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159895902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159895903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:15,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159895903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:15,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:15,936 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/d9bb87efd0c442ee8a3e592294c13120 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/d9bb87efd0c442ee8a3e592294c13120 2024-12-02T17:17:15,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742493_1669 (size=31105) 2024-12-02T17:17:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-02T17:17:15,947 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:15,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:15,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/9a301e8c138d4922bdea5e4d4c8ce81f 2024-12-02T17:17:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-02T17:17:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:15,960 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:15,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:15,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:15,976 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into d9bb87efd0c442ee8a3e592294c13120(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-02T17:17:15,976 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:15,976 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=12, startTime=1733159834891; duration=1sec 2024-12-02T17:17:15,976 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:15,976 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:16,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159896007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159896008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159896027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159896027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159896028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/dbf1442770b64d64b639f404f42393ac is 50, key is test_row_0/B:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:16,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742494_1670 (size=12151) 2024-12-02T17:17:16,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/dbf1442770b64d64b639f404f42393ac 2024-12-02T17:17:16,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/85fe11e6f65b4e9a91104e23a4be29e5 is 50, key is test_row_0/C:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:16,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742495_1671 (size=12151) 2024-12-02T17:17:16,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/85fe11e6f65b4e9a91104e23a4be29e5 2024-12-02T17:17:16,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/9a301e8c138d4922bdea5e4d4c8ce81f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f 2024-12-02T17:17:16,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f, entries=150, sequenceid=183, filesize=30.4 K 2024-12-02T17:17:16,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/dbf1442770b64d64b639f404f42393ac as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/dbf1442770b64d64b639f404f42393ac 2024-12-02T17:17:16,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/dbf1442770b64d64b639f404f42393ac, entries=150, sequenceid=183, filesize=11.9 K 2024-12-02T17:17:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/85fe11e6f65b4e9a91104e23a4be29e5 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/85fe11e6f65b4e9a91104e23a4be29e5 2024-12-02T17:17:16,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159896217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159896218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/85fe11e6f65b4e9a91104e23a4be29e5, entries=150, sequenceid=183, filesize=11.9 K 2024-12-02T17:17:16,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 305ba6d9a38ffb55509e7e10e4a765ff in 391ms, sequenceid=183, compaction requested=true 2024-12-02T17:17:16,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:16,241 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:16,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:16,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:16,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:16,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:16,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:16,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:16,242 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:16,244 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:16,244 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:16,244 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,244 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/d9bb87efd0c442ee8a3e592294c13120, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=91.4 K 2024-12-02T17:17:16,244 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,244 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/d9bb87efd0c442ee8a3e592294c13120, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f] 2024-12-02T17:17:16,245 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9bb87efd0c442ee8a3e592294c13120, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733159831334 2024-12-02T17:17:16,245 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743bd2bb46ce463eaa02a2f55a6dfbaa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733159833534 2024-12-02T17:17:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:16,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:17:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:16,251 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:16,251 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:16,251 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,251 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a301e8c138d4922bdea5e4d4c8ce81f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733159835792 2024-12-02T17:17:16,251 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d8ff06f15860467a8040903b573bf244, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/1125cb6a8f2f4d9e817fce1e767f3300, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/dbf1442770b64d64b639f404f42393ac] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=35.9 K 2024-12-02T17:17:16,254 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ff06f15860467a8040903b573bf244, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733159831334 2024-12-02T17:17:16,255 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 1125cb6a8f2f4d9e817fce1e767f3300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733159833534 2024-12-02T17:17:16,255 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf1442770b64d64b639f404f42393ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733159835792 2024-12-02T17:17:16,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021fcf4927ec7349e0897328af3f816a30_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159836244/Put/seqid=0 2024-12-02T17:17:16,273 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:16,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,285 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202b792470d48b0476ebdce6d60a9749c21_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,287 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202b792470d48b0476ebdce6d60a9749c21_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:16,287 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202b792470d48b0476ebdce6d60a9749c21_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:16,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742496_1672 (size=14794) 2024-12-02T17:17:16,294 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:16,298 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#575 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:16,299 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/906acbfdf9314c95a16accd7ac0d5fcb is 50, key is test_row_0/B:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:16,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159896289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159896298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159896305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,313 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021fcf4927ec7349e0897328af3f816a30_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021fcf4927ec7349e0897328af3f816a30_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:16,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/372d19e9ff604341a7cbc3d49140a139, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:16,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/372d19e9ff604341a7cbc3d49140a139 is 175, key is test_row_0/A:col10/1733159836244/Put/seqid=0 2024-12-02T17:17:16,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742497_1673 (size=4469) 2024-12-02T17:17:16,349 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#574 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:16,349 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f0ef06668e5c469fa6a9a26e39b3fc78 is 175, key is test_row_0/A:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:16,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742498_1674 (size=12561) 2024-12-02T17:17:16,381 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/906acbfdf9314c95a16accd7ac0d5fcb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/906acbfdf9314c95a16accd7ac0d5fcb 2024-12-02T17:17:16,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742499_1675 (size=39749) 2024-12-02T17:17:16,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742500_1676 (size=31515) 2024-12-02T17:17:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159896407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,412 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/372d19e9ff604341a7cbc3d49140a139 2024-12-02T17:17:16,415 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into 906acbfdf9314c95a16accd7ac0d5fcb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:16,415 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:16,415 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=13, startTime=1733159836242; duration=0sec 2024-12-02T17:17:16,415 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:16,415 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:16,415 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:16,417 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:16,418 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:16,418 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,418 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/992269e8a62f4f6a9057bb3f30fda279, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/5035ecf7978c4b69974ce38b89c15087, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/85fe11e6f65b4e9a91104e23a4be29e5] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=35.9 K 2024-12-02T17:17:16,419 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 992269e8a62f4f6a9057bb3f30fda279, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733159831334 2024-12-02T17:17:16,419 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 5035ecf7978c4b69974ce38b89c15087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733159833534 2024-12-02T17:17:16,419 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 85fe11e6f65b4e9a91104e23a4be29e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733159835792 2024-12-02T17:17:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159896417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159896417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/2d07f71635f843d3a06f38e07f944d00 is 50, key is test_row_0/B:col10/1733159836244/Put/seqid=0 2024-12-02T17:17:16,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,448 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#577 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:16,449 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/fa2c079da22c46df9525a282808ba2e9 is 50, key is test_row_0/C:col10/1733159835792/Put/seqid=0 2024-12-02T17:17:16,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742501_1677 (size=12151) 2024-12-02T17:17:16,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742502_1678 (size=12561) 2024-12-02T17:17:16,501 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/fa2c079da22c46df9525a282808ba2e9 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/fa2c079da22c46df9525a282808ba2e9 2024-12-02T17:17:16,512 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into fa2c079da22c46df9525a282808ba2e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:16,513 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:16,513 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=13, startTime=1733159836242; duration=0sec 2024-12-02T17:17:16,513 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:16,513 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159896537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159896537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:16,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159896620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159896637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159896637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,821 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/f0ef06668e5c469fa6a9a26e39b3fc78 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f0ef06668e5c469fa6a9a26e39b3fc78 2024-12-02T17:17:16,828 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into f0ef06668e5c469fa6a9a26e39b3fc78(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:16,828 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:16,828 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=13, startTime=1733159836241; duration=0sec 2024-12-02T17:17:16,828 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:16,828 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:16,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/2d07f71635f843d3a06f38e07f944d00 2024-12-02T17:17:16,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:16,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:16,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:16,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:16,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9cc22c82ff504310af32af587bf1ccf7 is 50, key is test_row_0/C:col10/1733159836244/Put/seqid=0 2024-12-02T17:17:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159896941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159896949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159896949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:16,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742503_1679 (size=12151) 2024-12-02T17:17:16,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9cc22c82ff504310af32af587bf1ccf7 2024-12-02T17:17:16,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/372d19e9ff604341a7cbc3d49140a139 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139 2024-12-02T17:17:16,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139, entries=200, sequenceid=203, filesize=38.8 K 2024-12-02T17:17:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/2d07f71635f843d3a06f38e07f944d00 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/2d07f71635f843d3a06f38e07f944d00 2024-12-02T17:17:16,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/2d07f71635f843d3a06f38e07f944d00, entries=150, sequenceid=203, filesize=11.9 K 2024-12-02T17:17:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9cc22c82ff504310af32af587bf1ccf7 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9cc22c82ff504310af32af587bf1ccf7 2024-12-02T17:17:16,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9cc22c82ff504310af32af587bf1ccf7, entries=150, sequenceid=203, filesize=11.9 K 2024-12-02T17:17:16,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 305ba6d9a38ffb55509e7e10e4a765ff in 735ms, sequenceid=203, compaction requested=false 2024-12-02T17:17:16,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:17,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:17:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:17,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:17,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120218164c0b44fa4dfb9e2d982313d36508_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159836286/Put/seqid=0 2024-12-02T17:17:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742504_1680 (size=14794) 2024-12-02T17:17:17,086 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:17,103 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120218164c0b44fa4dfb9e2d982313d36508_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120218164c0b44fa4dfb9e2d982313d36508_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:17,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/0da8e3fe4bdc4164be09ab3de1162228, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:17,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/0da8e3fe4bdc4164be09ab3de1162228 is 175, key is test_row_0/A:col10/1733159836286/Put/seqid=0 2024-12-02T17:17:17,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742505_1681 (size=39749) 2024-12-02T17:17:17,125 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/0da8e3fe4bdc4164be09ab3de1162228 2024-12-02T17:17:17,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/db8f80c77b1e4407b994b80775f055b6 is 50, key is test_row_0/B:col10/1733159836286/Put/seqid=0 2024-12-02T17:17:17,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742506_1682 (size=12151) 2024-12-02T17:17:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159897146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159897149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:17,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159897260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159897260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:17,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159897461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159897462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159897467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159897476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159897477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/db8f80c77b1e4407b994b80775f055b6 2024-12-02T17:17:17,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/d075ca0a1ba4418496a799ebd4bca7d8 is 50, key is test_row_0/C:col10/1733159836286/Put/seqid=0 2024-12-02T17:17:17,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742507_1683 (size=12151) 2024-12-02T17:17:17,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/d075ca0a1ba4418496a799ebd4bca7d8 2024-12-02T17:17:17,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/0da8e3fe4bdc4164be09ab3de1162228 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228 2024-12-02T17:17:17,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228, entries=200, sequenceid=220, filesize=38.8 K 2024-12-02T17:17:17,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/db8f80c77b1e4407b994b80775f055b6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/db8f80c77b1e4407b994b80775f055b6 2024-12-02T17:17:17,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/db8f80c77b1e4407b994b80775f055b6, entries=150, sequenceid=220, filesize=11.9 K 2024-12-02T17:17:17,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/d075ca0a1ba4418496a799ebd4bca7d8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/d075ca0a1ba4418496a799ebd4bca7d8 2024-12-02T17:17:17,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/d075ca0a1ba4418496a799ebd4bca7d8, entries=150, sequenceid=220, filesize=11.9 K 2024-12-02T17:17:17,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 305ba6d9a38ffb55509e7e10e4a765ff in 651ms, sequenceid=220, compaction requested=true 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:17,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:17:17,712 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:17,712 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:17,717 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:17,717 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:17,717 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,717 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/fa2c079da22c46df9525a282808ba2e9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9cc22c82ff504310af32af587bf1ccf7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/d075ca0a1ba4418496a799ebd4bca7d8] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.0 K 2024-12-02T17:17:17,717 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:17,717 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:17,717 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,717 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2c079da22c46df9525a282808ba2e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733159835792 2024-12-02T17:17:17,717 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f0ef06668e5c469fa6a9a26e39b3fc78, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=108.4 K 2024-12-02T17:17:17,717 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,717 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f0ef06668e5c469fa6a9a26e39b3fc78, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228] 2024-12-02T17:17:17,718 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc22c82ff504310af32af587bf1ccf7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733159835896 2024-12-02T17:17:17,718 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0ef06668e5c469fa6a9a26e39b3fc78, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733159835792 2024-12-02T17:17:17,718 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting d075ca0a1ba4418496a799ebd4bca7d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733159836286 2024-12-02T17:17:17,719 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 372d19e9ff604341a7cbc3d49140a139, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733159835896 2024-12-02T17:17:17,719 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da8e3fe4bdc4164be09ab3de1162228, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733159836281 2024-12-02T17:17:17,730 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#582 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:17,731 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/63fab916ae834944a8932a767a457fac is 50, key is test_row_0/C:col10/1733159836286/Put/seqid=0 2024-12-02T17:17:17,745 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:17,753 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120263653db23b334b6cbbeec8f96124bfd5_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:17,755 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120263653db23b334b6cbbeec8f96124bfd5_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:17,756 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120263653db23b334b6cbbeec8f96124bfd5_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:17,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742508_1684 (size=12663) 2024-12-02T17:17:17,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:17:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:17,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:17,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120287b7a7c727134fb1b0557eff66603b57_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159837131/Put/seqid=0 2024-12-02T17:17:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742509_1685 (size=4469) 2024-12-02T17:17:17,825 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#583 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:17,825 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/933395ddf5b24e11ae92cd17cc85a86a is 175, key is test_row_0/A:col10/1733159836286/Put/seqid=0 2024-12-02T17:17:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742510_1686 (size=14794) 2024-12-02T17:17:17,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:17,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159897833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120287b7a7c727134fb1b0557eff66603b57_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120287b7a7c727134fb1b0557eff66603b57_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:17,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159897839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/e1c7ea462cbc4fc386d6aa7567e17cd2, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:17,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/e1c7ea462cbc4fc386d6aa7567e17cd2 is 175, key is test_row_0/A:col10/1733159837131/Put/seqid=0 2024-12-02T17:17:17,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742511_1687 (size=31617) 2024-12-02T17:17:17,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:17,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:17,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:17,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742512_1688 (size=39749) 2024-12-02T17:17:17,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/e1c7ea462cbc4fc386d6aa7567e17cd2 2024-12-02T17:17:17,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/34794d3686584cf8b760767f628d7806 is 50, key is test_row_0/B:col10/1733159837131/Put/seqid=0 2024-12-02T17:17:17,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159897950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742513_1689 (size=12151) 2024-12-02T17:17:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/34794d3686584cf8b760767f628d7806 2024-12-02T17:17:17,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/41147aeee70b43cca47d6245dfa50c6d is 50, key is test_row_0/C:col10/1733159837131/Put/seqid=0 2024-12-02T17:17:17,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159897957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:17,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:17,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742514_1690 (size=12151) 2024-12-02T17:17:17,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/41147aeee70b43cca47d6245dfa50c6d 2024-12-02T17:17:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/e1c7ea462cbc4fc386d6aa7567e17cd2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2 2024-12-02T17:17:17,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2, entries=200, sequenceid=239, filesize=38.8 K 2024-12-02T17:17:18,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/34794d3686584cf8b760767f628d7806 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/34794d3686584cf8b760767f628d7806 2024-12-02T17:17:18,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/34794d3686584cf8b760767f628d7806, entries=150, sequenceid=239, filesize=11.9 K 2024-12-02T17:17:18,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/41147aeee70b43cca47d6245dfa50c6d as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/41147aeee70b43cca47d6245dfa50c6d 2024-12-02T17:17:18,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/41147aeee70b43cca47d6245dfa50c6d, entries=150, sequenceid=239, filesize=11.9 K 2024-12-02T17:17:18,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 305ba6d9a38ffb55509e7e10e4a765ff in 248ms, sequenceid=239, compaction requested=true 2024-12-02T17:17:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:18,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:17:18,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:18,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T17:17:18,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:18,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-02T17:17:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:18,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:18,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024c19228850e2424d80574c2f2ad8603b_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159837830/Put/seqid=0 2024-12-02T17:17:18,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742515_1691 (size=14794) 2024-12-02T17:17:18,235 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/63fab916ae834944a8932a767a457fac as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/63fab916ae834944a8932a767a457fac 2024-12-02T17:17:18,256 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into 63fab916ae834944a8932a767a457fac(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:18,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:18,256 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=13, startTime=1733159837712; duration=0sec 2024-12-02T17:17:18,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-02T17:17:18,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:18,257 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-02T17:17:18,259 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:17:18,259 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:17:18,259 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. because compaction request was cancelled 2024-12-02T17:17:18,259 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:18,259 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:18,259 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:17:18,261 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:17:18,261 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:18,261 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,261 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/906acbfdf9314c95a16accd7ac0d5fcb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/2d07f71635f843d3a06f38e07f944d00, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/db8f80c77b1e4407b994b80775f055b6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/34794d3686584cf8b760767f628d7806] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=47.9 K 2024-12-02T17:17:18,262 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 906acbfdf9314c95a16accd7ac0d5fcb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733159835792 2024-12-02T17:17:18,262 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d07f71635f843d3a06f38e07f944d00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733159835896 2024-12-02T17:17:18,263 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting db8f80c77b1e4407b994b80775f055b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733159836286 2024-12-02T17:17:18,263 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 34794d3686584cf8b760767f628d7806, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733159837129 2024-12-02T17:17:18,263 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/933395ddf5b24e11ae92cd17cc85a86a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/933395ddf5b24e11ae92cd17cc85a86a 2024-12-02T17:17:18,278 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into 933395ddf5b24e11ae92cd17cc85a86a(size=30.9 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:18,278 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=13, startTime=1733159837712; duration=0sec 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. because compaction request was cancelled 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:18,278 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:17:18,279 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:17:18,279 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:17:18,279 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. because compaction request was cancelled 2024-12-02T17:17:18,279 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:18,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159898273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159898286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,305 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#588 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:18,306 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/a0591aab6dce41d0aa98c13388cfa284 is 50, key is test_row_0/B:col10/1733159837131/Put/seqid=0 2024-12-02T17:17:18,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742516_1692 (size=12697) 2024-12-02T17:17:18,377 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/a0591aab6dce41d0aa98c13388cfa284 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/a0591aab6dce41d0aa98c13388cfa284 2024-12-02T17:17:18,400 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into a0591aab6dce41d0aa98c13388cfa284(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:18,400 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:18,400 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=12, startTime=1733159838039; duration=0sec 2024-12-02T17:17:18,401 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:18,401 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159898396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159898408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159898477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,481 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 305ba6d9a38ffb55509e7e10e4a765ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:17:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159898478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159898479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,484 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 305ba6d9a38ffb55509e7e10e4a765ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:17:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,488 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3b3375d1a925,40723,1733159666211 for 'test_row_2' in 305ba6d9a38ffb55509e7e10e4a765ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T17:17:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159898617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,628 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:18,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159898628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412024c19228850e2424d80574c2f2ad8603b_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024c19228850e2424d80574c2f2ad8603b_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:18,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/502eeae3c56c4644a459bb3fd3f50632, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:18,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/502eeae3c56c4644a459bb3fd3f50632 is 175, key is test_row_0/A:col10/1733159837830/Put/seqid=0 2024-12-02T17:17:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742517_1693 (size=39749) 2024-12-02T17:17:18,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/502eeae3c56c4644a459bb3fd3f50632 2024-12-02T17:17:18,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c6e116fc587f44bbba22f67f96b15b6c is 50, key is test_row_0/B:col10/1733159837830/Put/seqid=0 2024-12-02T17:17:18,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742518_1694 (size=12151) 2024-12-02T17:17:18,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c6e116fc587f44bbba22f67f96b15b6c 2024-12-02T17:17:18,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9c9a4043336e4fc69946ce2d0d1263ed is 50, key is test_row_0/C:col10/1733159837830/Put/seqid=0 2024-12-02T17:17:18,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742519_1695 (size=12151) 2024-12-02T17:17:18,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9c9a4043336e4fc69946ce2d0d1263ed 2024-12-02T17:17:18,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/502eeae3c56c4644a459bb3fd3f50632 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632 2024-12-02T17:17:18,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632, entries=200, sequenceid=253, filesize=38.8 K 2024-12-02T17:17:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c6e116fc587f44bbba22f67f96b15b6c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c6e116fc587f44bbba22f67f96b15b6c 2024-12-02T17:17:18,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c6e116fc587f44bbba22f67f96b15b6c, entries=150, sequenceid=253, filesize=11.9 K 2024-12-02T17:17:18,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9c9a4043336e4fc69946ce2d0d1263ed as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9c9a4043336e4fc69946ce2d0d1263ed 2024-12-02T17:17:18,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9c9a4043336e4fc69946ce2d0d1263ed, entries=150, sequenceid=253, filesize=11.9 K 2024-12-02T17:17:18,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 305ba6d9a38ffb55509e7e10e4a765ff in 767ms, sequenceid=253, compaction requested=true 2024-12-02T17:17:18,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:18,931 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:18,932 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:18,932 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:18,932 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,932 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/933395ddf5b24e11ae92cd17cc85a86a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=108.5 K 2024-12-02T17:17:18,932 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,933 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/933395ddf5b24e11ae92cd17cc85a86a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632] 2024-12-02T17:17:18,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:18,933 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933395ddf5b24e11ae92cd17cc85a86a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733159836286 2024-12-02T17:17:18,935 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c7ea462cbc4fc386d6aa7567e17cd2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733159837129 2024-12-02T17:17:18,936 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 502eeae3c56c4644a459bb3fd3f50632, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733159837794 2024-12-02T17:17:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:18,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:18,940 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:17:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:18,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-02T17:17:18,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:18,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:18,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:18,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:18,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:18,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:18,956 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:17:18,956 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:17:18,956 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. because compaction request was cancelled 2024-12-02T17:17:18,956 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:18,957 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:18,961 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:18,961 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:18,961 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,962 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/63fab916ae834944a8932a767a457fac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/41147aeee70b43cca47d6245dfa50c6d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9c9a4043336e4fc69946ce2d0d1263ed] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.1 K 2024-12-02T17:17:18,962 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 63fab916ae834944a8932a767a457fac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733159836286 2024-12-02T17:17:18,963 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 41147aeee70b43cca47d6245dfa50c6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733159837129 2024-12-02T17:17:18,964 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c9a4043336e4fc69946ce2d0d1263ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733159837830 2024-12-02T17:17:18,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:18,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,965 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:18,970 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202069ecfb54c8148269d4bba54c86cbf40_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:18,974 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202069ecfb54c8148269d4bba54c86cbf40_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:18,974 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202069ecfb54c8148269d4bba54c86cbf40_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:18,980 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#592 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:18,981 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/b1b2b5b80746461e88e660627829050e is 50, key is test_row_0/C:col10/1733159837830/Put/seqid=0 2024-12-02T17:17:18,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028ac5cc40c2a6439a96ccfe097924dce9_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159838934/Put/seqid=0 2024-12-02T17:17:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159898982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159898985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:18,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742521_1697 (size=12765) 2024-12-02T17:17:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742522_1698 (size=12454) 2024-12-02T17:17:19,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742520_1696 (size=4469) 2024-12-02T17:17:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159899097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159899106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:19,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:19,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159899307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159899316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:19,443 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#591 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:19,443 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/b1b2b5b80746461e88e660627829050e as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b1b2b5b80746461e88e660627829050e 2024-12-02T17:17:19,445 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/cd49bb60acf64626b8073e5d7f8d4b1b is 175, key is test_row_0/A:col10/1733159837830/Put/seqid=0 2024-12-02T17:17:19,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412028ac5cc40c2a6439a96ccfe097924dce9_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028ac5cc40c2a6439a96ccfe097924dce9_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:19,452 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/da76d20ad1384b49b66130368fa9e063, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:19,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/da76d20ad1384b49b66130368fa9e063 is 175, key is test_row_0/A:col10/1733159838934/Put/seqid=0 2024-12-02T17:17:19,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,473 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into b1b2b5b80746461e88e660627829050e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:19,473 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:19,474 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=13, startTime=1733159838940; duration=0sec 2024-12-02T17:17:19,474 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:19,474 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:19,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742524_1700 (size=31255) 2024-12-02T17:17:19,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/da76d20ad1384b49b66130368fa9e063 2024-12-02T17:17:19,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742523_1699 (size=31719) 2024-12-02T17:17:19,526 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/cd49bb60acf64626b8073e5d7f8d4b1b as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/cd49bb60acf64626b8073e5d7f8d4b1b 2024-12-02T17:17:19,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/aee8a52aadae43aa9f2d0c15bf88dfb6 is 50, key is test_row_0/B:col10/1733159838934/Put/seqid=0 2024-12-02T17:17:19,537 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into cd49bb60acf64626b8073e5d7f8d4b1b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:19,537 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:19,537 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=13, startTime=1733159838931; duration=0sec 2024-12-02T17:17:19,537 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:19,537 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:19,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742525_1701 (size=12301) 2024-12-02T17:17:19,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/aee8a52aadae43aa9f2d0c15bf88dfb6 2024-12-02T17:17:19,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/925b2b3cf67f462f982902dab544fde6 is 50, key is test_row_0/C:col10/1733159838934/Put/seqid=0 2024-12-02T17:17:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742526_1702 (size=12301) 2024-12-02T17:17:19,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/925b2b3cf67f462f982902dab544fde6 2024-12-02T17:17:19,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:19,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:19,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159899617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/da76d20ad1384b49b66130368fa9e063 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063 2024-12-02T17:17:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159899638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:19,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063, entries=150, sequenceid=275, filesize=30.5 K 2024-12-02T17:17:19,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/aee8a52aadae43aa9f2d0c15bf88dfb6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/aee8a52aadae43aa9f2d0c15bf88dfb6 2024-12-02T17:17:19,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/aee8a52aadae43aa9f2d0c15bf88dfb6, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T17:17:19,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/925b2b3cf67f462f982902dab544fde6 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/925b2b3cf67f462f982902dab544fde6 2024-12-02T17:17:19,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/925b2b3cf67f462f982902dab544fde6, entries=150, sequenceid=275, filesize=12.0 K 2024-12-02T17:17:19,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 305ba6d9a38ffb55509e7e10e4a765ff in 781ms, sequenceid=275, compaction requested=true 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:19,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:17:19,722 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:19,722 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:17:19,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:17:19,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:17:19,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. because compaction request was cancelled 2024-12-02T17:17:19,729 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:19,729 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-02T17:17:19,733 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:19,733 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:19,733 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,733 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/a0591aab6dce41d0aa98c13388cfa284, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c6e116fc587f44bbba22f67f96b15b6c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/aee8a52aadae43aa9f2d0c15bf88dfb6] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.3 K 2024-12-02T17:17:19,736 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-02T17:17:19,737 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-02T17:17:19,737 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. because compaction request was cancelled 2024-12-02T17:17:19,737 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:19,737 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0591aab6dce41d0aa98c13388cfa284, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733159837129 2024-12-02T17:17:19,752 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e116fc587f44bbba22f67f96b15b6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733159837830 2024-12-02T17:17:19,753 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting aee8a52aadae43aa9f2d0c15bf88dfb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733159838270 2024-12-02T17:17:19,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:19,777 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:19,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:19,795 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#596 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:19,796 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/9f073db88e804f43ba9d4cf95821adb1 is 50, key is test_row_0/B:col10/1733159838934/Put/seqid=0 2024-12-02T17:17:19,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c64f6d680c9c4f61a32c794013883d88_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159838982/Put/seqid=0 2024-12-02T17:17:19,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742527_1703 (size=12949) 2024-12-02T17:17:19,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742528_1704 (size=12454) 2024-12-02T17:17:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:19,859 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/9f073db88e804f43ba9d4cf95821adb1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/9f073db88e804f43ba9d4cf95821adb1 2024-12-02T17:17:19,863 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202c64f6d680c9c4f61a32c794013883d88_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c64f6d680c9c4f61a32c794013883d88_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7f5030ea25534f34a26f51011e859626, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:19,866 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into 9f073db88e804f43ba9d4cf95821adb1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:19,866 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:19,867 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=13, startTime=1733159839722; duration=0sec 2024-12-02T17:17:19,867 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:19,867 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7f5030ea25534f34a26f51011e859626 is 175, key is test_row_0/A:col10/1733159838982/Put/seqid=0 2024-12-02T17:17:19,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742529_1705 (size=31255) 2024-12-02T17:17:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:20,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159900237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159900237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,274 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7f5030ea25534f34a26f51011e859626 2024-12-02T17:17:20,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c1eebc1692104a94b5d3a0658aac96aa is 50, key is test_row_0/B:col10/1733159838982/Put/seqid=0 2024-12-02T17:17:20,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742530_1706 (size=12301) 2024-12-02T17:17:20,330 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c1eebc1692104a94b5d3a0658aac96aa 2024-12-02T17:17:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9b6e379b803e49958ac0108c20ae3975 is 50, key is test_row_0/C:col10/1733159838982/Put/seqid=0 2024-12-02T17:17:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159900347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159900357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742531_1707 (size=12301) 2024-12-02T17:17:20,370 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9b6e379b803e49958ac0108c20ae3975 2024-12-02T17:17:20,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7f5030ea25534f34a26f51011e859626 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626 2024-12-02T17:17:20,383 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626, entries=150, sequenceid=290, filesize=30.5 K 2024-12-02T17:17:20,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c1eebc1692104a94b5d3a0658aac96aa as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c1eebc1692104a94b5d3a0658aac96aa 2024-12-02T17:17:20,391 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c1eebc1692104a94b5d3a0658aac96aa, entries=150, sequenceid=290, filesize=12.0 K 2024-12-02T17:17:20,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/9b6e379b803e49958ac0108c20ae3975 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9b6e379b803e49958ac0108c20ae3975 2024-12-02T17:17:20,396 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9b6e379b803e49958ac0108c20ae3975, entries=150, sequenceid=290, filesize=12.0 K 2024-12-02T17:17:20,397 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 305ba6d9a38ffb55509e7e10e4a765ff in 620ms, sequenceid=290, compaction requested=true 2024-12-02T17:17:20,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:20,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:20,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-02T17:17:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-02T17:17:20,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-02T17:17:20,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4350 sec 2024-12-02T17:17:20,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 4.4520 sec 2024-12-02T17:17:20,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159840223, max timestamp=1733159840491Num uniques 859; 2024-12-02T17:17:20,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:20,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159840223, max timestamp=1733159840491Num uniques 859; 2024-12-02T17:17:20,495 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:20,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159840223, max timestamp=1733159840491Num uniques 859; 2024-12-02T17:17:20,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:20,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-02T17:17:20,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:20,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:20,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:20,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:20,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:20,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:20,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dd57595d45cf4857b3e4fabe7dd81181_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:20,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742532_1708 (size=17534) 2024-12-02T17:17:20,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159840546, max timestamp=1733159840572Num uniques 859; 2024-12-02T17:17:20,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:20,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159840546, max timestamp=1733159840572Num uniques 859; 2024-12-02T17:17:20,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:20,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159840546, max timestamp=1733159840572Num uniques 859; 2024-12-02T17:17:20,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159900613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159900615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159900620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159900620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159900622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159900729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159900729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159900737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159900737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159900741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159900947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159900948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159900951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159900957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159900957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:20,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:20,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202dd57595d45cf4857b3e4fabe7dd81181_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dd57595d45cf4857b3e4fabe7dd81181_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:20,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/4a0cec9893514544b12784ec38833f49, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:20,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/4a0cec9893514544b12784ec38833f49 is 175, key is test_row_0/A:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:21,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742533_1709 (size=48639) 2024-12-02T17:17:21,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/4a0cec9893514544b12784ec38833f49 2024-12-02T17:17:21,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c5826bfc26c54bd28d537c03415f73cd is 50, key is test_row_0/B:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:21,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742534_1710 (size=12301) 2024-12-02T17:17:21,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c5826bfc26c54bd28d537c03415f73cd 2024-12-02T17:17:21,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/b0667134db134ade9aa0455f12f9d430 is 50, key is test_row_0/C:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:21,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742535_1711 (size=12301) 2024-12-02T17:17:21,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/b0667134db134ade9aa0455f12f9d430 2024-12-02T17:17:21,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/4a0cec9893514544b12784ec38833f49 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49 2024-12-02T17:17:21,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49, entries=250, sequenceid=317, filesize=47.5 K 2024-12-02T17:17:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/c5826bfc26c54bd28d537c03415f73cd as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c5826bfc26c54bd28d537c03415f73cd 2024-12-02T17:17:21,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c5826bfc26c54bd28d537c03415f73cd, entries=150, sequenceid=317, filesize=12.0 K 2024-12-02T17:17:21,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/b0667134db134ade9aa0455f12f9d430 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b0667134db134ade9aa0455f12f9d430 2024-12-02T17:17:21,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b0667134db134ade9aa0455f12f9d430, entries=150, sequenceid=317, filesize=12.0 K 2024-12-02T17:17:21,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for 305ba6d9a38ffb55509e7e10e4a765ff in 746ms, sequenceid=317, compaction requested=true 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:21,254 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:17:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:21,254 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:21,255 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:21,255 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:21,255 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:21,255 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:17:21,255 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/9f073db88e804f43ba9d4cf95821adb1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c1eebc1692104a94b5d3a0658aac96aa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c5826bfc26c54bd28d537c03415f73cd] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.7 K 2024-12-02T17:17:21,255 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:21,255 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:21,255 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/cd49bb60acf64626b8073e5d7f8d4b1b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=139.5 K 2024-12-02T17:17:21,256 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:21,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/cd49bb60acf64626b8073e5d7f8d4b1b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49] 2024-12-02T17:17:21,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f073db88e804f43ba9d4cf95821adb1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733159838270 2024-12-02T17:17:21,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd49bb60acf64626b8073e5d7f8d4b1b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733159837830 2024-12-02T17:17:21,256 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c1eebc1692104a94b5d3a0658aac96aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733159838950 2024-12-02T17:17:21,256 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting da76d20ad1384b49b66130368fa9e063, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733159838270 2024-12-02T17:17:21,257 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting c5826bfc26c54bd28d537c03415f73cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733159840228 2024-12-02T17:17:21,257 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f5030ea25534f34a26f51011e859626, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733159838950 2024-12-02T17:17:21,257 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a0cec9893514544b12784ec38833f49, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733159840228 2024-12-02T17:17:21,263 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:21,265 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#603 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:21,265 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412026e0f3eca1d0b4ee6af9f8b71a5b84af9_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:21,266 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e234d7e968654d7fa9d93b56ce72daec is 50, key is test_row_0/B:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:21,268 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412026e0f3eca1d0b4ee6af9f8b71a5b84af9_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:21,268 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412026e0f3eca1d0b4ee6af9f8b71a5b84af9_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:21,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-02T17:17:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:21,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:21,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742536_1712 (size=13051) 2024-12-02T17:17:21,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159901290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159901291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159901296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159901297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159901306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202894f3cf9fe33481fb9854a059774d105_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159840614/Put/seqid=0 2024-12-02T17:17:21,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742537_1713 (size=4469) 2024-12-02T17:17:21,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742538_1714 (size=14994) 2024-12-02T17:17:21,357 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:21,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202894f3cf9fe33481fb9854a059774d105_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202894f3cf9fe33481fb9854a059774d105_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:21,375 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/bc44d8eda45d42049f36257b829a4309, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:21,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/bc44d8eda45d42049f36257b829a4309 is 175, key is test_row_0/A:col10/1733159840614/Put/seqid=0 2024-12-02T17:17:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159901417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159901417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159901417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159901416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742539_1715 (size=39949) 2024-12-02T17:17:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159901427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159901637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159901638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159901638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159901641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159901648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,712 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e234d7e968654d7fa9d93b56ce72daec as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e234d7e968654d7fa9d93b56ce72daec 2024-12-02T17:17:21,728 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into e234d7e968654d7fa9d93b56ce72daec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:21,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:21,728 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=13, startTime=1733159841254; duration=0sec 2024-12-02T17:17:21,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:21,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:21,728 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-02T17:17:21,729 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#604 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:21,730 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/2b16b7b3c33c4ad68de745ea86835297 is 175, key is test_row_0/A:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:21,736 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-02T17:17:21,736 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:21,736 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:21,736 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b1b2b5b80746461e88e660627829050e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/925b2b3cf67f462f982902dab544fde6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9b6e379b803e49958ac0108c20ae3975, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b0667134db134ade9aa0455f12f9d430] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=48.5 K 2024-12-02T17:17:21,736 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b2b5b80746461e88e660627829050e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733159837830 2024-12-02T17:17:21,737 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 925b2b3cf67f462f982902dab544fde6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733159838270 2024-12-02T17:17:21,737 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6e379b803e49958ac0108c20ae3975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733159838950 2024-12-02T17:17:21,737 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b0667134db134ade9aa0455f12f9d430, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733159840228 2024-12-02T17:17:21,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742540_1716 (size=32005) 2024-12-02T17:17:21,750 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#606 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:21,751 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/93d9a34377ee4e508831a44e4e3ee3d4 is 50, key is test_row_0/C:col10/1733159840228/Put/seqid=0 2024-12-02T17:17:21,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742541_1717 (size=13051) 2024-12-02T17:17:21,832 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/bc44d8eda45d42049f36257b829a4309 2024-12-02T17:17:21,833 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/93d9a34377ee4e508831a44e4e3ee3d4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/93d9a34377ee4e508831a44e4e3ee3d4 2024-12-02T17:17:21,865 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into 93d9a34377ee4e508831a44e4e3ee3d4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:21,865 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:21,865 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=12, startTime=1733159841254; duration=0sec 2024-12-02T17:17:21,865 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:21,865 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:21,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e5357ec4277b4440b525a7dec1328662 is 50, key is test_row_0/B:col10/1733159840614/Put/seqid=0 2024-12-02T17:17:21,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742542_1718 (size=12301) 2024-12-02T17:17:21,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159901957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159901957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159901959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159901959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:21,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159901969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:22,157 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/2b16b7b3c33c4ad68de745ea86835297 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/2b16b7b3c33c4ad68de745ea86835297 2024-12-02T17:17:22,169 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into 2b16b7b3c33c4ad68de745ea86835297(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:22,169 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:22,169 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=12, startTime=1733159841254; duration=0sec 2024-12-02T17:17:22,169 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:22,169 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:22,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e5357ec4277b4440b525a7dec1328662 2024-12-02T17:17:22,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/28655fb6f80144c5a9ad3c44043b789a is 50, key is test_row_0/C:col10/1733159840614/Put/seqid=0 2024-12-02T17:17:22,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742543_1719 (size=12301) 2024-12-02T17:17:22,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159902467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:22,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159902477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:22,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159902477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:22,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159902480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:22,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159902489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:22,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/28655fb6f80144c5a9ad3c44043b789a 2024-12-02T17:17:22,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/bc44d8eda45d42049f36257b829a4309 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309 2024-12-02T17:17:22,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309, entries=200, sequenceid=354, filesize=39.0 K 2024-12-02T17:17:22,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/e5357ec4277b4440b525a7dec1328662 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e5357ec4277b4440b525a7dec1328662 2024-12-02T17:17:22,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e5357ec4277b4440b525a7dec1328662, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T17:17:22,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/28655fb6f80144c5a9ad3c44043b789a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/28655fb6f80144c5a9ad3c44043b789a 2024-12-02T17:17:22,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/28655fb6f80144c5a9ad3c44043b789a, entries=150, sequenceid=354, filesize=12.0 K 2024-12-02T17:17:22,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 305ba6d9a38ffb55509e7e10e4a765ff in 1568ms, sequenceid=354, compaction requested=false 2024-12-02T17:17:22,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:23,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:17:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:23,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:23,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022e1107404ca8498985c5bd6abf182b5c_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:23,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742544_1720 (size=14994) 2024-12-02T17:17:23,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159843532, max timestamp=1733159843548Num uniques 859; 2024-12-02T17:17:23,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:17:23,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733159843532, max timestamp=1733159843550Num uniques 859; 2024-12-02T17:17:23,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-02T17:17:23,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:23,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733159843532, max timestamp=1733159843550Num uniques 859; 2024-12-02T17:17:23,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/3b3375d1a925:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-02T17:17:23,569 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412022e1107404ca8498985c5bd6abf182b5c_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022e1107404ca8498985c5bd6abf182b5c_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:23,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7b659eaecdfa48828e7f8c2d01613435, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:23,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7b659eaecdfa48828e7f8c2d01613435 is 175, key is test_row_0/A:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159903601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159903601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159903605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159903611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159903613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742545_1721 (size=39949) 2024-12-02T17:17:23,628 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7b659eaecdfa48828e7f8c2d01613435 2024-12-02T17:17:23,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/605dbbeab3fd4e75b7c5c7f88add054c is 50, key is test_row_0/B:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742546_1722 (size=12301) 2024-12-02T17:17:23,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/605dbbeab3fd4e75b7c5c7f88add054c 2024-12-02T17:17:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159903717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159903717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159903717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159903729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159903729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/939e5ae8b25946efb8019125b876665f is 50, key is test_row_0/C:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:23,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742547_1723 (size=12301) 2024-12-02T17:17:23,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/939e5ae8b25946efb8019125b876665f 2024-12-02T17:17:23,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7b659eaecdfa48828e7f8c2d01613435 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435 2024-12-02T17:17:23,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435, entries=200, sequenceid=368, filesize=39.0 K 2024-12-02T17:17:23,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/605dbbeab3fd4e75b7c5c7f88add054c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/605dbbeab3fd4e75b7c5c7f88add054c 2024-12-02T17:17:23,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/605dbbeab3fd4e75b7c5c7f88add054c, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T17:17:23,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/939e5ae8b25946efb8019125b876665f as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/939e5ae8b25946efb8019125b876665f 2024-12-02T17:17:23,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/939e5ae8b25946efb8019125b876665f, entries=150, sequenceid=368, filesize=12.0 K 2024-12-02T17:17:23,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 305ba6d9a38ffb55509e7e10e4a765ff in 410ms, sequenceid=368, compaction requested=true 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:23,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-02T17:17:23,902 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:23,904 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:23,925 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:23,925 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:23,925 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:23,925 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/93d9a34377ee4e508831a44e4e3ee3d4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/28655fb6f80144c5a9ad3c44043b789a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/939e5ae8b25946efb8019125b876665f] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.8 K 2024-12-02T17:17:23,925 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:23,925 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:23,925 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:23,925 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/2b16b7b3c33c4ad68de745ea86835297, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=109.3 K 2024-12-02T17:17:23,925 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:23,925 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/2b16b7b3c33c4ad68de745ea86835297, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435] 2024-12-02T17:17:23,932 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b16b7b3c33c4ad68de745ea86835297, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733159840228 2024-12-02T17:17:23,933 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d9a34377ee4e508831a44e4e3ee3d4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733159840228 2024-12-02T17:17:23,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-02T17:17:23,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:23,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:23,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:23,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:23,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:23,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-02T17:17:23,936 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28655fb6f80144c5a9ad3c44043b789a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733159840614 2024-12-02T17:17:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:23,937 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting bc44d8eda45d42049f36257b829a4309, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733159840614 2024-12-02T17:17:23,944 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939e5ae8b25946efb8019125b876665f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733159841285 2024-12-02T17:17:23,945 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b659eaecdfa48828e7f8c2d01613435, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733159841285 2024-12-02T17:17:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159903948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159903949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159903950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159903959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159903960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:23,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f6fc50f3cf994f91a163d9f6a31f9c0e_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159843600/Put/seqid=0 2024-12-02T17:17:23,981 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:23,983 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#613 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:23,983 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/79ac5c06bb904f6387e896e6eb1b94f2 is 50, key is test_row_0/C:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:23,983 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202804309d1e94d405081f14d2a12910b4a_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:23,986 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202804309d1e94d405081f14d2a12910b4a_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:23,986 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202804309d1e94d405081f14d2a12910b4a_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742548_1724 (size=14994) 2024-12-02T17:17:24,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742549_1725 (size=13153) 2024-12-02T17:17:24,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742550_1726 (size=4469) 2024-12-02T17:17:24,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159904067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159904068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159904068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159904077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159904078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,088 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/79ac5c06bb904f6387e896e6eb1b94f2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/79ac5c06bb904f6387e896e6eb1b94f2 2024-12-02T17:17:24,114 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into 79ac5c06bb904f6387e896e6eb1b94f2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:24,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:24,114 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=13, startTime=1733159843902; duration=0sec 2024-12-02T17:17:24,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:24,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:24,114 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:24,115 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:24,115 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:24,115 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,115 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e234d7e968654d7fa9d93b56ce72daec, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e5357ec4277b4440b525a7dec1328662, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/605dbbeab3fd4e75b7c5c7f88add054c] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.8 K 2024-12-02T17:17:24,115 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e234d7e968654d7fa9d93b56ce72daec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733159840228 2024-12-02T17:17:24,121 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5357ec4277b4440b525a7dec1328662, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733159840614 2024-12-02T17:17:24,121 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 605dbbeab3fd4e75b7c5c7f88add054c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733159841285 2024-12-02T17:17:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T17:17:24,147 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:24,149 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-02T17:17:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-02T17:17:24,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:24,155 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-02T17:17:24,158 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#615 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:24,158 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/53874cc0c47045009845993c3c9d2ac1 is 50, key is test_row_0/B:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:24,159 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T17:17:24,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T17:17:24,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742551_1727 (size=13153) 2024-12-02T17:17:24,247 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T17:17:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159904277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159904277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159904287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159904293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159904297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,410 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:24,433 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202f6fc50f3cf994f91a163d9f6a31f9c0e_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f6fc50f3cf994f91a163d9f6a31f9c0e_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:24,452 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/56eaff8500d649b9bf5ab4888a478991, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:24,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/56eaff8500d649b9bf5ab4888a478991 is 175, key is test_row_0/A:col10/1733159843600/Put/seqid=0 2024-12-02T17:17:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:24,473 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#614 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:24,474 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/6ef83c1cdfd54ebb8d7d1ef3757e3308 is 175, key is test_row_0/A:col10/1733159843490/Put/seqid=0 2024-12-02T17:17:24,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:24,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742552_1728 (size=39949) 2024-12-02T17:17:24,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742553_1729 (size=32107) 2024-12-02T17:17:24,536 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/6ef83c1cdfd54ebb8d7d1ef3757e3308 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/6ef83c1cdfd54ebb8d7d1ef3757e3308 2024-12-02T17:17:24,547 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into 6ef83c1cdfd54ebb8d7d1ef3757e3308(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:24,547 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:24,548 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=13, startTime=1733159843902; duration=0sec 2024-12-02T17:17:24,548 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:24,548 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53240 deadline: 1733159904587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159904588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,608 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/53874cc0c47045009845993c3c9d2ac1 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/53874cc0c47045009845993c3c9d2ac1 2024-12-02T17:17:24,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159904607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53164 deadline: 1733159904607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53232 deadline: 1733159904609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:24,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:24,617 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into 53874cc0c47045009845993c3c9d2ac1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:24,617 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:24,617 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=13, startTime=1733159843902; duration=0sec 2024-12-02T17:17:24,617 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:24,617 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:24,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:24,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:24,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:24,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:24,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/56eaff8500d649b9bf5ab4888a478991 2024-12-02T17:17:24,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/91adfc29ae0d40539ba0399328b71c0a is 50, key is test_row_0/B:col10/1733159843600/Put/seqid=0 2024-12-02T17:17:24,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742554_1730 (size=12301) 2024-12-02T17:17:24,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/91adfc29ae0d40539ba0399328b71c0a 2024-12-02T17:17:24,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:24,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:24,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/0708f816e04b4ec9a35601262542d99a is 50, key is test_row_0/C:col10/1733159843600/Put/seqid=0 2024-12-02T17:17:25,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742555_1731 (size=12301) 2024-12-02T17:17:25,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/0708f816e04b4ec9a35601262542d99a 2024-12-02T17:17:25,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/56eaff8500d649b9bf5ab4888a478991 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991 2024-12-02T17:17:25,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991, entries=200, sequenceid=405, filesize=39.0 K 2024-12-02T17:17:25,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/91adfc29ae0d40539ba0399328b71c0a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/91adfc29ae0d40539ba0399328b71c0a 2024-12-02T17:17:25,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/91adfc29ae0d40539ba0399328b71c0a, entries=150, sequenceid=405, filesize=12.0 K 2024-12-02T17:17:25,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/0708f816e04b4ec9a35601262542d99a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0708f816e04b4ec9a35601262542d99a 2024-12-02T17:17:25,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0708f816e04b4ec9a35601262542d99a, entries=150, sequenceid=405, filesize=12.0 K 2024-12-02T17:17:25,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=46.96 KB/48090 for 305ba6d9a38ffb55509e7e10e4a765ff in 1103ms, sequenceid=405, compaction requested=false 2024-12-02T17:17:25,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:25,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-02T17:17:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:25,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:25,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:25,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020a4af4a04a034a19990bb994448604da_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:25,176 INFO [Thread-2754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2754. 2024-12-02T17:17:25,177 DEBUG [Thread-2754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,177 DEBUG [Thread-2754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,177 DEBUG [Thread-2754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,177 INFO [Thread-2741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2741. 2024-12-02T17:17:25,178 DEBUG [Thread-2741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,178 DEBUG [Thread-2741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,178 DEBUG [Thread-2741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,180 INFO [Thread-2747 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2747. 2024-12-02T17:17:25,180 INFO [Thread-2743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2743. 2024-12-02T17:17:25,180 DEBUG [Thread-2743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,180 DEBUG [Thread-2743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,180 DEBUG [Thread-2743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,180 DEBUG [Thread-2747 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,180 DEBUG [Thread-2747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,181 DEBUG [Thread-2747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,183 INFO [Thread-2756 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2756. 2024-12-02T17:17:25,183 DEBUG [Thread-2756 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,183 DEBUG [Thread-2756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,183 DEBUG [Thread-2756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742556_1732 (size=17534) 2024-12-02T17:17:25,190 INFO [Thread-2758 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2758. 2024-12-02T17:17:25,190 DEBUG [Thread-2758 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,190 DEBUG [Thread-2758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,190 DEBUG [Thread-2758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159905173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,191 INFO [Thread-2752 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2752. 2024-12-02T17:17:25,191 DEBUG [Thread-2752 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,191 DEBUG [Thread-2752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,191 DEBUG [Thread-2752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,192 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:25,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159905175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,203 INFO [Thread-2750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2750. 2024-12-02T17:17:25,203 DEBUG [Thread-2750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:25,203 DEBUG [Thread-2750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,203 DEBUG [Thread-2750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:25,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:25,208 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412020a4af4a04a034a19990bb994448604da_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020a4af4a04a034a19990bb994448604da_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:25,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7fad3319213040a7a97876e59eb17aa3, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:25,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7fad3319213040a7a97876e59eb17aa3 is 175, key is test_row_0/A:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:25,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742557_1733 (size=48639) 2024-12-02T17:17:25,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:25,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159905297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159905297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159905507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159905507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:25,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,640 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7fad3319213040a7a97876e59eb17aa3 2024-12-02T17:17:25,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/b0992078a9c3402ab26f128ae5371563 is 50, key is test_row_0/B:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742558_1734 (size=12301) 2024-12-02T17:17:25,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159905817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159905817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:25,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:25,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:26,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/b0992078a9c3402ab26f128ae5371563 2024-12-02T17:17:26,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/262186af8e9f406cbd836d7c05be8ace is 50, key is test_row_0/C:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742559_1735 (size=12301) 2024-12-02T17:17:26,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53228 deadline: 1733159906327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:26,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T17:17:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53186 deadline: 1733159906328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 , the old value is region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T17:17:26,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., hostname=3b3375d1a925,40723,1733159666211, seqNum=5 because the exception is null or not the one we care about 2024-12-02T17:17:26,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:26,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:26,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T17:17:26,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/262186af8e9f406cbd836d7c05be8ace 2024-12-02T17:17:26,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/7fad3319213040a7a97876e59eb17aa3 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3 2024-12-02T17:17:26,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3, entries=250, sequenceid=420, filesize=47.5 K 2024-12-02T17:17:26,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/b0992078a9c3402ab26f128ae5371563 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/b0992078a9c3402ab26f128ae5371563 2024-12-02T17:17:26,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/b0992078a9c3402ab26f128ae5371563, entries=150, sequenceid=420, filesize=12.0 K 2024-12-02T17:17:26,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/262186af8e9f406cbd836d7c05be8ace as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/262186af8e9f406cbd836d7c05be8ace 2024-12-02T17:17:26,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/262186af8e9f406cbd836d7c05be8ace, entries=150, sequenceid=420, filesize=12.0 K 2024-12-02T17:17:26,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 305ba6d9a38ffb55509e7e10e4a765ff in 1380ms, sequenceid=420, compaction requested=true 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:A, priority=-2147483648, current under compaction store size is 1 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:B, priority=-2147483648, current under compaction store size is 2 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:26,492 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 305ba6d9a38ffb55509e7e10e4a765ff:C, priority=-2147483648, current under compaction store size is 3 2024-12-02T17:17:26,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-02T17:17:26,492 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/A is initiating minor compaction (all files) 2024-12-02T17:17:26,493 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/A in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:26,493 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/6ef83c1cdfd54ebb8d7d1ef3757e3308, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=117.9 K 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/B is initiating minor compaction (all files) 2024-12-02T17:17:26,493 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. files: [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/6ef83c1cdfd54ebb8d7d1ef3757e3308, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3] 2024-12-02T17:17:26,493 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/B in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,493 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/53874cc0c47045009845993c3c9d2ac1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/91adfc29ae0d40539ba0399328b71c0a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/b0992078a9c3402ab26f128ae5371563] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.9 K 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ef83c1cdfd54ebb8d7d1ef3757e3308, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733159841285 2024-12-02T17:17:26,493 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 53874cc0c47045009845993c3c9d2ac1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733159841285 2024-12-02T17:17:26,494 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56eaff8500d649b9bf5ab4888a478991, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733159843600 2024-12-02T17:17:26,494 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 91adfc29ae0d40539ba0399328b71c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733159843600 2024-12-02T17:17:26,494 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fad3319213040a7a97876e59eb17aa3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733159843946 2024-12-02T17:17:26,494 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting b0992078a9c3402ab26f128ae5371563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733159843946 2024-12-02T17:17:26,501 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#B#compaction#621 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:26,501 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:26,501 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/03927cb28287420d8ee200870cfc92e2 is 50, key is test_row_0/B:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:26,503 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241202193258b373924fefb8525b60f64f327a_305ba6d9a38ffb55509e7e10e4a765ff store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:26,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40723 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:26,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742560_1736 (size=13255) 2024-12-02T17:17:26,510 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241202193258b373924fefb8525b60f64f327a_305ba6d9a38ffb55509e7e10e4a765ff, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:26,510 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202193258b373924fefb8525b60f64f327a_305ba6d9a38ffb55509e7e10e4a765ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021626f0b7bd0d44cf89c607cfb0201f81_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159845175/Put/seqid=0 2024-12-02T17:17:26,527 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/03927cb28287420d8ee200870cfc92e2 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/03927cb28287420d8ee200870cfc92e2 2024-12-02T17:17:26,532 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/B of 305ba6d9a38ffb55509e7e10e4a765ff into 03927cb28287420d8ee200870cfc92e2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:26,532 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:26,532 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/B, priority=13, startTime=1733159846492; duration=0sec 2024-12-02T17:17:26,532 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-02T17:17:26,532 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:B 2024-12-02T17:17:26,532 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T17:17:26,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T17:17:26,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1541): 305ba6d9a38ffb55509e7e10e4a765ff/C is initiating minor compaction (all files) 2024-12-02T17:17:26,533 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 305ba6d9a38ffb55509e7e10e4a765ff/C in TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:26,533 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/79ac5c06bb904f6387e896e6eb1b94f2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0708f816e04b4ec9a35601262542d99a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/262186af8e9f406cbd836d7c05be8ace] into tmpdir=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp, totalSize=36.9 K 2024-12-02T17:17:26,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 79ac5c06bb904f6387e896e6eb1b94f2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733159841285 2024-12-02T17:17:26,533 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 0708f816e04b4ec9a35601262542d99a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733159843600 2024-12-02T17:17:26,534 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] compactions.Compactor(225): Compacting 262186af8e9f406cbd836d7c05be8ace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1733159843946 2024-12-02T17:17:26,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742561_1737 (size=4469) 2024-12-02T17:17:26,537 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#A#compaction#622 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:26,537 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/37c788695aff49188ad8abb6d2746bde is 175, key is test_row_0/A:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:26,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742562_1738 (size=12454) 2024-12-02T17:17:26,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:26,547 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 305ba6d9a38ffb55509e7e10e4a765ff#C#compaction#624 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T17:17:26,548 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/560206a5bf82499cad0937656d9d2a53 is 50, key is test_row_0/C:col10/1733159845107/Put/seqid=0 2024-12-02T17:17:26,553 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412021626f0b7bd0d44cf89c607cfb0201f81_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021626f0b7bd0d44cf89c607cfb0201f81_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:26,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/368e82dc57ab484680f1cb5f633d3b65, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/368e82dc57ab484680f1cb5f633d3b65 is 175, key is test_row_0/A:col10/1733159845175/Put/seqid=0 2024-12-02T17:17:26,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742563_1739 (size=32209) 2024-12-02T17:17:26,569 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/37c788695aff49188ad8abb6d2746bde as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/37c788695aff49188ad8abb6d2746bde 2024-12-02T17:17:26,574 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/A of 305ba6d9a38ffb55509e7e10e4a765ff into 37c788695aff49188ad8abb6d2746bde(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:26,574 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:26,574 INFO [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/A, priority=13, startTime=1733159846492; duration=0sec 2024-12-02T17:17:26,574 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:26,574 DEBUG [RS:0;3b3375d1a925:40723-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:A 2024-12-02T17:17:26,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742564_1740 (size=13255) 2024-12-02T17:17:26,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742565_1741 (size=31255) 2024-12-02T17:17:26,983 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/560206a5bf82499cad0937656d9d2a53 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/560206a5bf82499cad0937656d9d2a53 2024-12-02T17:17:26,987 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 305ba6d9a38ffb55509e7e10e4a765ff/C of 305ba6d9a38ffb55509e7e10e4a765ff into 560206a5bf82499cad0937656d9d2a53(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T17:17:26,987 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:26,987 INFO [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., storeName=305ba6d9a38ffb55509e7e10e4a765ff/C, priority=13, startTime=1733159846492; duration=0sec 2024-12-02T17:17:26,987 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T17:17:26,987 DEBUG [RS:0;3b3375d1a925:40723-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 305ba6d9a38ffb55509e7e10e4a765ff:C 2024-12-02T17:17:26,987 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/368e82dc57ab484680f1cb5f633d3b65 2024-12-02T17:17:26,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/20d0281e51894ef68fa9b5e8e26ceb28 is 50, key is test_row_0/B:col10/1733159845175/Put/seqid=0 2024-12-02T17:17:26,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742566_1742 (size=12301) 2024-12-02T17:17:26,999 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/20d0281e51894ef68fa9b5e8e26ceb28 2024-12-02T17:17:27,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/4bd3c26d5ef14426bc5c40f929264bbb is 50, key is test_row_0/C:col10/1733159845175/Put/seqid=0 2024-12-02T17:17:27,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742567_1743 (size=12301) 2024-12-02T17:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40723 {}] regionserver.HRegion(8855): Flush requested on 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:27,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. as already flushing 2024-12-02T17:17:27,339 INFO [Thread-2739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2739. 2024-12-02T17:17:27,339 DEBUG [Thread-2739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:27,339 DEBUG [Thread-2739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:27,339 INFO [Thread-2745 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2745. 2024-12-02T17:17:27,339 DEBUG [Thread-2745 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-02T17:17:27,339 DEBUG [Thread-2745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:27,339 DEBUG [Thread-2739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:27,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:27,339 DEBUG [Thread-2745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:27,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:27,410 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/4bd3c26d5ef14426bc5c40f929264bbb 2024-12-02T17:17:27,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/368e82dc57ab484680f1cb5f633d3b65 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/368e82dc57ab484680f1cb5f633d3b65 2024-12-02T17:17:27,418 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/368e82dc57ab484680f1cb5f633d3b65, entries=150, sequenceid=440, filesize=30.5 K 2024-12-02T17:17:27,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/20d0281e51894ef68fa9b5e8e26ceb28 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/20d0281e51894ef68fa9b5e8e26ceb28 2024-12-02T17:17:27,422 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/20d0281e51894ef68fa9b5e8e26ceb28, entries=150, sequenceid=440, filesize=12.0 K 2024-12-02T17:17:27,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/4bd3c26d5ef14426bc5c40f929264bbb as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/4bd3c26d5ef14426bc5c40f929264bbb 2024-12-02T17:17:27,426 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/4bd3c26d5ef14426bc5c40f929264bbb, entries=150, sequenceid=440, filesize=12.0 K 2024-12-02T17:17:27,429 INFO [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 305ba6d9a38ffb55509e7e10e4a765ff in 923ms, sequenceid=440, compaction requested=false 2024-12-02T17:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 305ba6d9a38ffb55509e7e10e4a765ff: 2024-12-02T17:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:27,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3b3375d1a925:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-02T17:17:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-02T17:17:27,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-02T17:17:27,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2710 sec 2024-12-02T17:17:27,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.2830 sec 2024-12-02T17:17:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-02T17:17:28,307 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1946 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1961 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1939 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1947 2024-12-02T17:17:28,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1963 2024-12-02T17:17:28,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-02T17:17:28,308 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:17:28,308 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:17:28,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:28,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:28,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:28,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-02T17:17:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T17:17:28,312 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159848312"}]},"ts":"1733159848312"} 2024-12-02T17:17:28,314 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-02T17:17:28,314 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-02T17:17:28,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-02T17:17:28,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, UNASSIGN}] 2024-12-02T17:17:28,317 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, UNASSIGN 2024-12-02T17:17:28,317 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=CLOSING, regionLocation=3b3375d1a925,40723,1733159666211 2024-12-02T17:17:28,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, UNASSIGN because future has completed 2024-12-02T17:17:28,319 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T17:17:28,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211}] 2024-12-02T17:17:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T17:17:28,471 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:28,471 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 305ba6d9a38ffb55509e7e10e4a765ff, disabling compactions & flushes 2024-12-02T17:17:28,472 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. after waiting 0 ms 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:28,472 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 305ba6d9a38ffb55509e7e10e4a765ff 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=A 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=B 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 305ba6d9a38ffb55509e7e10e4a765ff, store=C 2024-12-02T17:17:28,472 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-02T17:17:28,477 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cc99473400214460ba2bcaab6cb09f56_305ba6d9a38ffb55509e7e10e4a765ff is 50, key is test_row_0/A:col10/1733159847337/Put/seqid=0 2024-12-02T17:17:28,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742568_1744 (size=9914) 2024-12-02T17:17:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T17:17:28,881 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T17:17:28,884 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241202cc99473400214460ba2bcaab6cb09f56_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cc99473400214460ba2bcaab6cb09f56_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:28,885 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/fb35bbb8f5144b07aa1fa8de38630ba8, store: [table=TestAcidGuarantees family=A region=305ba6d9a38ffb55509e7e10e4a765ff] 2024-12-02T17:17:28,886 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/fb35bbb8f5144b07aa1fa8de38630ba8 is 175, key is test_row_0/A:col10/1733159847337/Put/seqid=0 2024-12-02T17:17:28,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742569_1745 (size=22561) 2024-12-02T17:17:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T17:17:29,290 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=448, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/fb35bbb8f5144b07aa1fa8de38630ba8 2024-12-02T17:17:29,296 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/d1dd32112f5d4dc18ec739859b405314 is 50, key is test_row_0/B:col10/1733159847337/Put/seqid=0 2024-12-02T17:17:29,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742570_1746 (size=9857) 2024-12-02T17:17:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T17:17:29,700 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/d1dd32112f5d4dc18ec739859b405314 2024-12-02T17:17:29,706 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/c610c8e11eb64915987b8f55226f4ab4 is 50, key is test_row_0/C:col10/1733159847337/Put/seqid=0 2024-12-02T17:17:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742571_1747 (size=9857) 2024-12-02T17:17:30,109 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/c610c8e11eb64915987b8f55226f4ab4 2024-12-02T17:17:30,113 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/A/fb35bbb8f5144b07aa1fa8de38630ba8 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/fb35bbb8f5144b07aa1fa8de38630ba8 2024-12-02T17:17:30,117 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/fb35bbb8f5144b07aa1fa8de38630ba8, entries=100, sequenceid=448, filesize=22.0 K 2024-12-02T17:17:30,117 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/B/d1dd32112f5d4dc18ec739859b405314 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d1dd32112f5d4dc18ec739859b405314 2024-12-02T17:17:30,121 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d1dd32112f5d4dc18ec739859b405314, entries=100, sequenceid=448, filesize=9.6 K 2024-12-02T17:17:30,121 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/.tmp/C/c610c8e11eb64915987b8f55226f4ab4 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/c610c8e11eb64915987b8f55226f4ab4 2024-12-02T17:17:30,124 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/c610c8e11eb64915987b8f55226f4ab4, entries=100, sequenceid=448, filesize=9.6 K 2024-12-02T17:17:30,125 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 305ba6d9a38ffb55509e7e10e4a765ff in 1653ms, sequenceid=448, compaction requested=true 2024-12-02T17:17:30,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ac9521f807044925a66f80a2d28773d7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/d9bb87efd0c442ee8a3e592294c13120, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f0ef06668e5c469fa6a9a26e39b3fc78, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/933395ddf5b24e11ae92cd17cc85a86a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/cd49bb60acf64626b8073e5d7f8d4b1b, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/2b16b7b3c33c4ad68de745ea86835297, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/6ef83c1cdfd54ebb8d7d1ef3757e3308, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3] to archive 2024-12-02T17:17:30,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:17:30,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ead07f93900e4895a94b94ab1425f429 2024-12-02T17:17:30,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/566f9a20cedc467fbbd66a43947f8a60 2024-12-02T17:17:30,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f1c5ed27397c4c139588bc482076c484 2024-12-02T17:17:30,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/c40ad5d87ec94c21ac692f744e719a07 2024-12-02T17:17:30,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/aded406dc1dd44608022431ce40cd2be 2024-12-02T17:17:30,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ac9521f807044925a66f80a2d28773d7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ac9521f807044925a66f80a2d28773d7 2024-12-02T17:17:30,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/1cb1caf82e5a4d599a0a7bb5f8168995 2024-12-02T17:17:30,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/ee1e3e2042b84bfc93a725e56910487f 2024-12-02T17:17:30,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f726c91bfe164a88bd58bc8ef537d336 2024-12-02T17:17:30,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/d9bb87efd0c442ee8a3e592294c13120 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/d9bb87efd0c442ee8a3e592294c13120 2024-12-02T17:17:30,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/743bd2bb46ce463eaa02a2f55a6dfbaa 2024-12-02T17:17:30,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f0ef06668e5c469fa6a9a26e39b3fc78 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/f0ef06668e5c469fa6a9a26e39b3fc78 2024-12-02T17:17:30,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/9a301e8c138d4922bdea5e4d4c8ce81f 2024-12-02T17:17:30,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/372d19e9ff604341a7cbc3d49140a139 2024-12-02T17:17:30,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/0da8e3fe4bdc4164be09ab3de1162228 2024-12-02T17:17:30,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/933395ddf5b24e11ae92cd17cc85a86a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/933395ddf5b24e11ae92cd17cc85a86a 2024-12-02T17:17:30,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/e1c7ea462cbc4fc386d6aa7567e17cd2 2024-12-02T17:17:30,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/502eeae3c56c4644a459bb3fd3f50632 2024-12-02T17:17:30,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/cd49bb60acf64626b8073e5d7f8d4b1b to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/cd49bb60acf64626b8073e5d7f8d4b1b 2024-12-02T17:17:30,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/da76d20ad1384b49b66130368fa9e063 2024-12-02T17:17:30,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7f5030ea25534f34a26f51011e859626 2024-12-02T17:17:30,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/4a0cec9893514544b12784ec38833f49 2024-12-02T17:17:30,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/2b16b7b3c33c4ad68de745ea86835297 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/2b16b7b3c33c4ad68de745ea86835297 2024-12-02T17:17:30,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/bc44d8eda45d42049f36257b829a4309 2024-12-02T17:17:30,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7b659eaecdfa48828e7f8c2d01613435 2024-12-02T17:17:30,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/6ef83c1cdfd54ebb8d7d1ef3757e3308 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/6ef83c1cdfd54ebb8d7d1ef3757e3308 2024-12-02T17:17:30,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/56eaff8500d649b9bf5ab4888a478991 2024-12-02T17:17:30,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/7fad3319213040a7a97876e59eb17aa3 2024-12-02T17:17:30,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/77d2c25416934eef91e9ef5087a6f9e7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/15d2d08a358644c0aa61a8ec280062f2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/00e0973846d74b08a2988e1febd79039, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e0ff749f75574eff8aa54c5e86e22336, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/bcd32e2f262442e7a62baf6d8ee58616, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/abfa07c4e4ee453189f0a0008100e0ab, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0888356376404c9ba1bd5dc4a683d689, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/045ca978596e47bb9747dae68e03271f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d8ff06f15860467a8040903b573bf244, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0018f87381eb49b9a621474c2eafdae4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/1125cb6a8f2f4d9e817fce1e767f3300, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/906acbfdf9314c95a16accd7ac0d5fcb, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/dbf1442770b64d64b639f404f42393ac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/2d07f71635f843d3a06f38e07f944d00, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/db8f80c77b1e4407b994b80775f055b6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/a0591aab6dce41d0aa98c13388cfa284, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/34794d3686584cf8b760767f628d7806, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c6e116fc587f44bbba22f67f96b15b6c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/9f073db88e804f43ba9d4cf95821adb1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/aee8a52aadae43aa9f2d0c15bf88dfb6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c1eebc1692104a94b5d3a0658aac96aa, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e234d7e968654d7fa9d93b56ce72daec, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c5826bfc26c54bd28d537c03415f73cd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e5357ec4277b4440b525a7dec1328662, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/53874cc0c47045009845993c3c9d2ac1, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/605dbbeab3fd4e75b7c5c7f88add054c, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/91adfc29ae0d40539ba0399328b71c0a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/b0992078a9c3402ab26f128ae5371563] to archive 2024-12-02T17:17:30,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:17:30,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/77d2c25416934eef91e9ef5087a6f9e7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/77d2c25416934eef91e9ef5087a6f9e7 2024-12-02T17:17:30,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/15d2d08a358644c0aa61a8ec280062f2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/15d2d08a358644c0aa61a8ec280062f2 2024-12-02T17:17:30,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/00e0973846d74b08a2988e1febd79039 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/00e0973846d74b08a2988e1febd79039 2024-12-02T17:17:30,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e0ff749f75574eff8aa54c5e86e22336 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e0ff749f75574eff8aa54c5e86e22336 2024-12-02T17:17:30,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/bcd32e2f262442e7a62baf6d8ee58616 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/bcd32e2f262442e7a62baf6d8ee58616 2024-12-02T17:17:30,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/abfa07c4e4ee453189f0a0008100e0ab to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/abfa07c4e4ee453189f0a0008100e0ab 2024-12-02T17:17:30,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0888356376404c9ba1bd5dc4a683d689 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0888356376404c9ba1bd5dc4a683d689 2024-12-02T17:17:30,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/045ca978596e47bb9747dae68e03271f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/045ca978596e47bb9747dae68e03271f 2024-12-02T17:17:30,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d8ff06f15860467a8040903b573bf244 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d8ff06f15860467a8040903b573bf244 2024-12-02T17:17:30,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0018f87381eb49b9a621474c2eafdae4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/0018f87381eb49b9a621474c2eafdae4 2024-12-02T17:17:30,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/1125cb6a8f2f4d9e817fce1e767f3300 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/1125cb6a8f2f4d9e817fce1e767f3300 2024-12-02T17:17:30,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/906acbfdf9314c95a16accd7ac0d5fcb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/906acbfdf9314c95a16accd7ac0d5fcb 2024-12-02T17:17:30,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/dbf1442770b64d64b639f404f42393ac to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/dbf1442770b64d64b639f404f42393ac 2024-12-02T17:17:30,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/2d07f71635f843d3a06f38e07f944d00 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/2d07f71635f843d3a06f38e07f944d00 2024-12-02T17:17:30,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/db8f80c77b1e4407b994b80775f055b6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/db8f80c77b1e4407b994b80775f055b6 2024-12-02T17:17:30,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/a0591aab6dce41d0aa98c13388cfa284 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/a0591aab6dce41d0aa98c13388cfa284 2024-12-02T17:17:30,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/34794d3686584cf8b760767f628d7806 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/34794d3686584cf8b760767f628d7806 2024-12-02T17:17:30,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c6e116fc587f44bbba22f67f96b15b6c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c6e116fc587f44bbba22f67f96b15b6c 2024-12-02T17:17:30,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/9f073db88e804f43ba9d4cf95821adb1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/9f073db88e804f43ba9d4cf95821adb1 2024-12-02T17:17:30,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/aee8a52aadae43aa9f2d0c15bf88dfb6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/aee8a52aadae43aa9f2d0c15bf88dfb6 2024-12-02T17:17:30,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c1eebc1692104a94b5d3a0658aac96aa to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c1eebc1692104a94b5d3a0658aac96aa 2024-12-02T17:17:30,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e234d7e968654d7fa9d93b56ce72daec to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e234d7e968654d7fa9d93b56ce72daec 2024-12-02T17:17:30,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c5826bfc26c54bd28d537c03415f73cd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/c5826bfc26c54bd28d537c03415f73cd 2024-12-02T17:17:30,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e5357ec4277b4440b525a7dec1328662 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/e5357ec4277b4440b525a7dec1328662 2024-12-02T17:17:30,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/53874cc0c47045009845993c3c9d2ac1 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/53874cc0c47045009845993c3c9d2ac1 2024-12-02T17:17:30,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/605dbbeab3fd4e75b7c5c7f88add054c to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/605dbbeab3fd4e75b7c5c7f88add054c 2024-12-02T17:17:30,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/91adfc29ae0d40539ba0399328b71c0a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/91adfc29ae0d40539ba0399328b71c0a 2024-12-02T17:17:30,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/b0992078a9c3402ab26f128ae5371563 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/b0992078a9c3402ab26f128ae5371563 2024-12-02T17:17:30,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/6709e62bc52e4221b3330a93824c7212, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/cdffde5a32674de7b11d048e923365e4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/447e39371b34454b9784194e52559f02, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/db476286b77540b8a32263046f8a0bb6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/df18c97a69184f9dbb91a539eedb9fbd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/11d65a1085af45269b05ab4fbe5669fd, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/ceb8a9837378461c82bfc66d957f7ef9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0d35e281bc2f442ea2fe8b6cafab4b43, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/992269e8a62f4f6a9057bb3f30fda279, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/870e72fc0fa74e99861df96dab6c8e10, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/5035ecf7978c4b69974ce38b89c15087, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/fa2c079da22c46df9525a282808ba2e9, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/85fe11e6f65b4e9a91104e23a4be29e5, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9cc22c82ff504310af32af587bf1ccf7, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/63fab916ae834944a8932a767a457fac, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/d075ca0a1ba4418496a799ebd4bca7d8, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/41147aeee70b43cca47d6245dfa50c6d, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b1b2b5b80746461e88e660627829050e, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9c9a4043336e4fc69946ce2d0d1263ed, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/925b2b3cf67f462f982902dab544fde6, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9b6e379b803e49958ac0108c20ae3975, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/93d9a34377ee4e508831a44e4e3ee3d4, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b0667134db134ade9aa0455f12f9d430, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/28655fb6f80144c5a9ad3c44043b789a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/79ac5c06bb904f6387e896e6eb1b94f2, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/939e5ae8b25946efb8019125b876665f, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0708f816e04b4ec9a35601262542d99a, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/262186af8e9f406cbd836d7c05be8ace] to archive 2024-12-02T17:17:30,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T17:17:30,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/6709e62bc52e4221b3330a93824c7212 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/6709e62bc52e4221b3330a93824c7212 2024-12-02T17:17:30,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/cdffde5a32674de7b11d048e923365e4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/cdffde5a32674de7b11d048e923365e4 2024-12-02T17:17:30,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/447e39371b34454b9784194e52559f02 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/447e39371b34454b9784194e52559f02 2024-12-02T17:17:30,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/db476286b77540b8a32263046f8a0bb6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/db476286b77540b8a32263046f8a0bb6 2024-12-02T17:17:30,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/df18c97a69184f9dbb91a539eedb9fbd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/df18c97a69184f9dbb91a539eedb9fbd 2024-12-02T17:17:30,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/11d65a1085af45269b05ab4fbe5669fd to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/11d65a1085af45269b05ab4fbe5669fd 2024-12-02T17:17:30,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/ceb8a9837378461c82bfc66d957f7ef9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/ceb8a9837378461c82bfc66d957f7ef9 2024-12-02T17:17:30,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0d35e281bc2f442ea2fe8b6cafab4b43 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0d35e281bc2f442ea2fe8b6cafab4b43 2024-12-02T17:17:30,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/992269e8a62f4f6a9057bb3f30fda279 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/992269e8a62f4f6a9057bb3f30fda279 2024-12-02T17:17:30,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/870e72fc0fa74e99861df96dab6c8e10 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/870e72fc0fa74e99861df96dab6c8e10 2024-12-02T17:17:30,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/5035ecf7978c4b69974ce38b89c15087 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/5035ecf7978c4b69974ce38b89c15087 2024-12-02T17:17:30,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/fa2c079da22c46df9525a282808ba2e9 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/fa2c079da22c46df9525a282808ba2e9 2024-12-02T17:17:30,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/85fe11e6f65b4e9a91104e23a4be29e5 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/85fe11e6f65b4e9a91104e23a4be29e5 2024-12-02T17:17:30,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9cc22c82ff504310af32af587bf1ccf7 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9cc22c82ff504310af32af587bf1ccf7 2024-12-02T17:17:30,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/63fab916ae834944a8932a767a457fac to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/63fab916ae834944a8932a767a457fac 2024-12-02T17:17:30,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/d075ca0a1ba4418496a799ebd4bca7d8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/d075ca0a1ba4418496a799ebd4bca7d8 2024-12-02T17:17:30,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/41147aeee70b43cca47d6245dfa50c6d to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/41147aeee70b43cca47d6245dfa50c6d 2024-12-02T17:17:30,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b1b2b5b80746461e88e660627829050e to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b1b2b5b80746461e88e660627829050e 2024-12-02T17:17:30,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9c9a4043336e4fc69946ce2d0d1263ed to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9c9a4043336e4fc69946ce2d0d1263ed 2024-12-02T17:17:30,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/925b2b3cf67f462f982902dab544fde6 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/925b2b3cf67f462f982902dab544fde6 2024-12-02T17:17:30,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9b6e379b803e49958ac0108c20ae3975 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/9b6e379b803e49958ac0108c20ae3975 2024-12-02T17:17:30,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/93d9a34377ee4e508831a44e4e3ee3d4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/93d9a34377ee4e508831a44e4e3ee3d4 2024-12-02T17:17:30,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b0667134db134ade9aa0455f12f9d430 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/b0667134db134ade9aa0455f12f9d430 2024-12-02T17:17:30,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/28655fb6f80144c5a9ad3c44043b789a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/28655fb6f80144c5a9ad3c44043b789a 2024-12-02T17:17:30,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/79ac5c06bb904f6387e896e6eb1b94f2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/79ac5c06bb904f6387e896e6eb1b94f2 2024-12-02T17:17:30,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/939e5ae8b25946efb8019125b876665f to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/939e5ae8b25946efb8019125b876665f 2024-12-02T17:17:30,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0708f816e04b4ec9a35601262542d99a to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/0708f816e04b4ec9a35601262542d99a 2024-12-02T17:17:30,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/262186af8e9f406cbd836d7c05be8ace to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/262186af8e9f406cbd836d7c05be8ace 2024-12-02T17:17:30,211 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/recovered.edits/451.seqid, newMaxSeqId=451, maxSeqId=4 2024-12-02T17:17:30,212 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. 2024-12-02T17:17:30,212 DEBUG [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 305ba6d9a38ffb55509e7e10e4a765ff: Waiting for close lock at 1733159848471Running coprocessor pre-close hooks at 1733159848471Disabling compacts and flushes for region at 1733159848471Disabling writes for close at 1733159848472 (+1 ms)Obtaining lock to block concurrent updates at 1733159848472Preparing flush snapshotting stores in 305ba6d9a38ffb55509e7e10e4a765ff at 1733159848472Finished memstore snapshotting TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733159848472Flushing stores of TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff. at 1733159848473 (+1 ms)Flushing 305ba6d9a38ffb55509e7e10e4a765ff/A: creating writer at 1733159848473Flushing mob file 305ba6d9a38ffb55509e7e10e4a765ff/A: appending metadata at 1733159848477 (+4 ms)Flushing mob file 305ba6d9a38ffb55509e7e10e4a765ff/A: closing flushed file at 1733159848477Flushing 305ba6d9a38ffb55509e7e10e4a765ff/A: appending metadata at 1733159848885 (+408 ms)Flushing 305ba6d9a38ffb55509e7e10e4a765ff/A: closing flushed file at 1733159848885Flushing 305ba6d9a38ffb55509e7e10e4a765ff/B: creating writer at 1733159849294 (+409 ms)Flushing 305ba6d9a38ffb55509e7e10e4a765ff/B: appending metadata at 1733159849296 (+2 ms)Flushing 305ba6d9a38ffb55509e7e10e4a765ff/B: closing flushed file at 1733159849296Flushing 305ba6d9a38ffb55509e7e10e4a765ff/C: creating writer at 1733159849703 (+407 ms)Flushing 305ba6d9a38ffb55509e7e10e4a765ff/C: appending metadata at 1733159849705 (+2 ms)Flushing 305ba6d9a38ffb55509e7e10e4a765ff/C: closing flushed file at 1733159849705Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32631a6a: reopening flushed file at 1733159850113 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fd432ab: reopening flushed file at 1733159850117 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f7e156e: reopening flushed file at 1733159850121 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 305ba6d9a38ffb55509e7e10e4a765ff in 1653ms, sequenceid=448, compaction requested=true at 1733159850125 (+4 ms)Writing region close event to WAL at 1733159850208 (+83 ms)Running coprocessor post-close hooks at 1733159850212 (+4 ms)Closed at 1733159850212 2024-12-02T17:17:30,214 INFO [RS_CLOSE_REGION-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,214 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=305ba6d9a38ffb55509e7e10e4a765ff, regionState=CLOSED 2024-12-02T17:17:30,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 because future has completed 2024-12-02T17:17:30,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-02T17:17:30,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 305ba6d9a38ffb55509e7e10e4a765ff, server=3b3375d1a925,40723,1733159666211 in 1.8970 sec 2024-12-02T17:17:30,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-12-02T17:17:30,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=305ba6d9a38ffb55509e7e10e4a765ff, UNASSIGN in 1.9020 sec 2024-12-02T17:17:30,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-02T17:17:30,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9060 sec 2024-12-02T17:17:30,223 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733159850222"}]},"ts":"1733159850222"} 2024-12-02T17:17:30,224 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-02T17:17:30,224 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-02T17:17:30,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9150 sec 2024-12-02T17:17:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T17:17:30,457 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:30,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-02T17:17:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:30,458 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T17:17:30,459 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:30,461 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,462 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C, FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/recovered.edits] 2024-12-02T17:17:30,464 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/368e82dc57ab484680f1cb5f633d3b65 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/368e82dc57ab484680f1cb5f633d3b65 2024-12-02T17:17:30,465 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/37c788695aff49188ad8abb6d2746bde to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/37c788695aff49188ad8abb6d2746bde 2024-12-02T17:17:30,466 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/fb35bbb8f5144b07aa1fa8de38630ba8 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/A/fb35bbb8f5144b07aa1fa8de38630ba8 2024-12-02T17:17:30,467 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/03927cb28287420d8ee200870cfc92e2 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/03927cb28287420d8ee200870cfc92e2 2024-12-02T17:17:30,468 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/20d0281e51894ef68fa9b5e8e26ceb28 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/20d0281e51894ef68fa9b5e8e26ceb28 2024-12-02T17:17:30,469 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d1dd32112f5d4dc18ec739859b405314 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/B/d1dd32112f5d4dc18ec739859b405314 2024-12-02T17:17:30,470 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/4bd3c26d5ef14426bc5c40f929264bbb to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/4bd3c26d5ef14426bc5c40f929264bbb 2024-12-02T17:17:30,471 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/560206a5bf82499cad0937656d9d2a53 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/560206a5bf82499cad0937656d9d2a53 2024-12-02T17:17:30,471 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/c610c8e11eb64915987b8f55226f4ab4 to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/C/c610c8e11eb64915987b8f55226f4ab4 2024-12-02T17:17:30,473 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/recovered.edits/451.seqid to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff/recovered.edits/451.seqid 2024-12-02T17:17:30,474 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/default/TestAcidGuarantees/305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,474 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-02T17:17:30,474 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T17:17:30,474 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-02T17:17:30,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020a4af4a04a034a19990bb994448604da_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412020a4af4a04a034a19990bb994448604da_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,477 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021626f0b7bd0d44cf89c607cfb0201f81_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021626f0b7bd0d44cf89c607cfb0201f81_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,478 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120218164c0b44fa4dfb9e2d982313d36508_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120218164c0b44fa4dfb9e2d982313d36508_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,479 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202195ef9455d4f4540a55ab1cd6f63005e_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202195ef9455d4f4540a55ab1cd6f63005e_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,479 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021fcf4927ec7349e0897328af3f816a30_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412021fcf4927ec7349e0897328af3f816a30_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,480 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202261187f20b0144c39b6dfd3bb41a0863_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202261187f20b0144c39b6dfd3bb41a0863_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,481 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022e1107404ca8498985c5bd6abf182b5c_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412022e1107404ca8498985c5bd6abf182b5c_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,482 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024c19228850e2424d80574c2f2ad8603b_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412024c19228850e2424d80574c2f2ad8603b_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,482 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120277270236efdb46548dbdf3785e925334_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120277270236efdb46548dbdf3785e925334_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,483 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028561825beac9456e96d323acc5c1601d_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028561825beac9456e96d323acc5c1601d_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,484 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120287b7a7c727134fb1b0557eff66603b57_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120287b7a7c727134fb1b0557eff66603b57_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,484 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202894f3cf9fe33481fb9854a059774d105_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202894f3cf9fe33481fb9854a059774d105_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,485 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028ac5cc40c2a6439a96ccfe097924dce9_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028ac5cc40c2a6439a96ccfe097924dce9_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028ede4f08a38a403190f6ded4cb384efa_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412028ede4f08a38a403190f6ded4cb384efa_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,487 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202917bd5e8e2c44ea8808436ae53350bcd_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202917bd5e8e2c44ea8808436ae53350bcd_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,488 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ab3384bc6ae4c03b0d0feaf147200da_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412029ab3384bc6ae4c03b0d0feaf147200da_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,489 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c64f6d680c9c4f61a32c794013883d88_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c64f6d680c9c4f61a32c794013883d88_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,490 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7f62f3695824ec38b211ec1908d0cdd_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202c7f62f3695824ec38b211ec1908d0cdd_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,491 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cbed7d5497ff47f4b31f4b57c221aa2f_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cbed7d5497ff47f4b31f4b57c221aa2f_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,491 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cc99473400214460ba2bcaab6cb09f56_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202cc99473400214460ba2bcaab6cb09f56_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,492 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d06845d749b841deba5d76f65d7b3449_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202d06845d749b841deba5d76f65d7b3449_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,493 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dd57595d45cf4857b3e4fabe7dd81181_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202dd57595d45cf4857b3e4fabe7dd81181_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,494 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f6fc50f3cf994f91a163d9f6a31f9c0e_305ba6d9a38ffb55509e7e10e4a765ff to hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241202f6fc50f3cf994f91a163d9f6a31f9c0e_305ba6d9a38ffb55509e7e10e4a765ff 2024-12-02T17:17:30,495 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-02T17:17:30,496 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:30,498 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-02T17:17:30,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-02T17:17:30,501 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:30,501 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-02T17:17:30,501 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733159850501"}]},"ts":"9223372036854775807"} 2024-12-02T17:17:30,503 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-02T17:17:30,503 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 305ba6d9a38ffb55509e7e10e4a765ff, NAME => 'TestAcidGuarantees,,1733159822066.305ba6d9a38ffb55509e7e10e4a765ff.', STARTKEY => '', ENDKEY => ''}] 2024-12-02T17:17:30,503 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-02T17:17:30,503 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733159850503"}]},"ts":"9223372036854775807"} 2024-12-02T17:17:30,504 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-02T17:17:30,505 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-02T17:17:30,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-12-02T17:17:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41221 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-02T17:17:30,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-02T17:17:30,567 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-02T17:17:30,576 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 249) - Thread LEAK? -, OpenFileDescriptor=453 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1000 (was 1067), ProcessCount=11 (was 11), AvailableMemoryMB=1538 (was 1704) 2024-12-02T17:17:30,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T17:17:30,576 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:17:30,576 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:17:30,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:30,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:30,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T17:17:30,576 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T17:17:30,576 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1603076298, stopped=false 2024-12-02T17:17:30,577 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3b3375d1a925,41221,1733159665139 2024-12-02T17:17:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T17:17:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T17:17:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:17:30,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:17:30,579 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:17:30,580 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:17:30,580 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T17:17:30,580 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:17:30,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:30,580 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T17:17:30,580 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3b3375d1a925,40723,1733159666211' ***** 2024-12-02T17:17:30,580 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T17:17:30,580 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T17:17:30,581 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(959): stopping server 3b3375d1a925,40723,1733159666211 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3b3375d1a925:40723. 2024-12-02T17:17:30,581 DEBUG [RS:0;3b3375d1a925:40723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T17:17:30,581 DEBUG [RS:0;3b3375d1a925:40723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T17:17:30,581 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T17:17:30,582 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T17:17:30,582 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T17:17:30,582 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T17:17:30,582 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T17:17:30,582 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T17:17:30,582 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T17:17:30,582 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T17:17:30,582 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T17:17:30,583 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-02T17:17:30,604 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/info/a6b0d563a9b545d3947bd654b956df85 is 95, key is TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c./info:/1733159702238/DeleteFamily/seqid=0 2024-12-02T17:17:30,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742572_1748 (size=6514) 2024-12-02T17:17:30,619 INFO [regionserver/3b3375d1a925:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:17:30,783 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T17:17:30,806 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T17:17:30,806 INFO [regionserver/3b3375d1a925:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T17:17:30,983 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T17:17:31,007 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/info/a6b0d563a9b545d3947bd654b956df85 2024-12-02T17:17:31,025 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/ns/5d07af65bf8d4549a1224127c1faba5c is 93, key is TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c./ns:/1733159702191/DeleteFamily/seqid=0 2024-12-02T17:17:31,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742573_1749 (size=5906) 2024-12-02T17:17:31,029 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/ns/5d07af65bf8d4549a1224127c1faba5c 2024-12-02T17:17:31,049 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/rep_barrier/7d30a298f5f7493ebe765276decc8410 is 102, key is TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c./rep_barrier:/1733159702191/DeleteFamily/seqid=0 2024-12-02T17:17:31,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742574_1750 (size=6025) 2024-12-02T17:17:31,183 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T17:17:31,383 DEBUG [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T17:17:31,453 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/rep_barrier/7d30a298f5f7493ebe765276decc8410 2024-12-02T17:17:31,475 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/table/38c38456c6e14760a8725dfd0d93d90a is 96, key is TestAcidGuarantees,,1733159669974.dcecfb16214ccdbe1c27643e3ed0b42c./table:/1733159702191/DeleteFamily/seqid=0 2024-12-02T17:17:31,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742575_1751 (size=5936) 2024-12-02T17:17:31,478 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/table/38c38456c6e14760a8725dfd0d93d90a 2024-12-02T17:17:31,482 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/info/a6b0d563a9b545d3947bd654b956df85 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/info/a6b0d563a9b545d3947bd654b956df85 2024-12-02T17:17:31,486 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/info/a6b0d563a9b545d3947bd654b956df85, entries=12, sequenceid=90, filesize=6.4 K 2024-12-02T17:17:31,487 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/ns/5d07af65bf8d4549a1224127c1faba5c as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/ns/5d07af65bf8d4549a1224127c1faba5c 2024-12-02T17:17:31,491 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/ns/5d07af65bf8d4549a1224127c1faba5c, entries=8, sequenceid=90, filesize=5.8 K 2024-12-02T17:17:31,492 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/rep_barrier/7d30a298f5f7493ebe765276decc8410 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/rep_barrier/7d30a298f5f7493ebe765276decc8410 2024-12-02T17:17:31,495 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/rep_barrier/7d30a298f5f7493ebe765276decc8410, entries=6, sequenceid=90, filesize=5.9 K 2024-12-02T17:17:31,496 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/.tmp/table/38c38456c6e14760a8725dfd0d93d90a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/table/38c38456c6e14760a8725dfd0d93d90a 2024-12-02T17:17:31,499 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/table/38c38456c6e14760a8725dfd0d93d90a, entries=7, sequenceid=90, filesize=5.8 K 2024-12-02T17:17:31,499 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 917ms, sequenceid=90, compaction requested=false 2024-12-02T17:17:31,507 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-02T17:17:31,508 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T17:17:31,508 INFO [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T17:17:31,508 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733159850582Running coprocessor pre-close hooks at 1733159850582Disabling compacts and flushes for region at 1733159850582Disabling writes for close at 1733159850582Obtaining lock to block concurrent updates at 1733159850583 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733159850583Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733159850583Flushing stores of hbase:meta,,1.1588230740 at 1733159850583Flushing 1588230740/info: creating writer at 1733159850584 (+1 ms)Flushing 1588230740/info: appending metadata at 1733159850603 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733159850603Flushing 1588230740/ns: creating writer at 1733159851010 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733159851025 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733159851025Flushing 1588230740/rep_barrier: creating writer at 1733159851033 (+8 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733159851049 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733159851049Flushing 1588230740/table: creating writer at 1733159851457 (+408 ms)Flushing 1588230740/table: appending metadata at 1733159851474 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733159851474Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d5b44ef: reopening flushed file at 1733159851482 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16489516: reopening flushed file at 1733159851486 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@afb10d9: reopening flushed file at 1733159851491 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b4549df: reopening flushed file at 1733159851495 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 917ms, sequenceid=90, compaction requested=false at 1733159851499 (+4 ms)Writing region close event to WAL at 1733159851505 (+6 ms)Running coprocessor post-close hooks at 1733159851507 (+2 ms)Closed at 1733159851508 (+1 ms) 2024-12-02T17:17:31,508 DEBUG [RS_CLOSE_META-regionserver/3b3375d1a925:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T17:17:31,583 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(976): stopping server 3b3375d1a925,40723,1733159666211; all regions closed. 2024-12-02T17:17:31,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741834_1010 (size=25344) 2024-12-02T17:17:31,589 DEBUG [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/oldWALs 2024-12-02T17:17:31,589 INFO [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b3375d1a925%2C40723%2C1733159666211.meta:.meta(num 1733159669385) 2024-12-02T17:17:31,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741833_1009 (size=19934693) 2024-12-02T17:17:31,592 DEBUG [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/oldWALs 2024-12-02T17:17:31,593 INFO [RS:0;3b3375d1a925:40723 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3b3375d1a925%2C40723%2C1733159666211:(num 1733159669029) 2024-12-02T17:17:31,593 DEBUG [RS:0;3b3375d1a925:40723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T17:17:31,593 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T17:17:31,593 INFO [RS:0;3b3375d1a925:40723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T17:17:31,593 INFO [RS:0;3b3375d1a925:40723 {}] hbase.ChoreService(370): Chore service for: regionserver/3b3375d1a925:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T17:17:31,593 INFO [RS:0;3b3375d1a925:40723 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T17:17:31,593 INFO [regionserver/3b3375d1a925:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T17:17:31,594 INFO [RS:0;3b3375d1a925:40723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40723 2024-12-02T17:17:31,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3b3375d1a925,40723,1733159666211 2024-12-02T17:17:31,600 INFO [RS:0;3b3375d1a925:40723 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T17:17:31,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T17:17:31,602 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3b3375d1a925,40723,1733159666211] 2024-12-02T17:17:31,604 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3b3375d1a925,40723,1733159666211 already deleted, retry=false 2024-12-02T17:17:31,604 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3b3375d1a925,40723,1733159666211 expired; onlineServers=0 2024-12-02T17:17:31,604 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3b3375d1a925,41221,1733159665139' ***** 2024-12-02T17:17:31,604 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T17:17:31,604 INFO [M:0;3b3375d1a925:41221 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T17:17:31,604 INFO [M:0;3b3375d1a925:41221 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T17:17:31,604 DEBUG [M:0;3b3375d1a925:41221 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T17:17:31,604 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T17:17:31,604 DEBUG [M:0;3b3375d1a925:41221 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T17:17:31,604 DEBUG [master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.large.0-1733159668555 {}] cleaner.HFileCleaner(306): Exit Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.large.0-1733159668555,5,FailOnTimeoutGroup] 2024-12-02T17:17:31,604 INFO [M:0;3b3375d1a925:41221 {}] hbase.ChoreService(370): Chore service for: master/3b3375d1a925:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T17:17:31,604 INFO [M:0;3b3375d1a925:41221 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T17:17:31,605 DEBUG [M:0;3b3375d1a925:41221 {}] master.HMaster(1795): Stopping service threads 2024-12-02T17:17:31,605 DEBUG [master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.small.0-1733159668556 {}] cleaner.HFileCleaner(306): Exit Thread[master/3b3375d1a925:0:becomeActiveMaster-HFileCleaner.small.0-1733159668556,5,FailOnTimeoutGroup] 2024-12-02T17:17:31,605 INFO [M:0;3b3375d1a925:41221 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T17:17:31,605 INFO [M:0;3b3375d1a925:41221 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T17:17:31,605 ERROR [M:0;3b3375d1a925:41221 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-02T17:17:31,605 INFO [M:0;3b3375d1a925:41221 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T17:17:31,606 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T17:17:31,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T17:17:31,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T17:17:31,607 DEBUG [M:0;3b3375d1a925:41221 {}] zookeeper.ZKUtil(347): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T17:17:31,607 WARN [M:0;3b3375d1a925:41221 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T17:17:31,607 INFO [M:0;3b3375d1a925:41221 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/.lastflushedseqids 2024-12-02T17:17:31,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742576_1752 (size=72) 2024-12-02T17:17:31,617 INFO [M:0;3b3375d1a925:41221 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T17:17:31,617 INFO [M:0;3b3375d1a925:41221 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T17:17:31,617 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T17:17:31,617 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:17:31,617 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:17:31,617 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T17:17:31,617 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:17:31,617 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=806.89 KB heapSize=991.31 KB 2024-12-02T17:17:31,632 DEBUG [M:0;3b3375d1a925:41221 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c399c93420344215b85e1e5b3329a784 is 82, key is hbase:meta,,1/info:regioninfo/1733159669541/Put/seqid=0 2024-12-02T17:17:31,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742577_1753 (size=5672) 2024-12-02T17:17:31,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:17:31,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40723-0x1009bfbc5620001, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:17:31,703 INFO [RS:0;3b3375d1a925:40723 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T17:17:31,703 INFO [RS:0;3b3375d1a925:40723 {}] regionserver.HRegionServer(1031): Exiting; stopping=3b3375d1a925,40723,1733159666211; zookeeper connection closed. 2024-12-02T17:17:31,703 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2bd82e6c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2bd82e6c 2024-12-02T17:17:31,704 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T17:17:32,035 INFO [M:0;3b3375d1a925:41221 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2290 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c399c93420344215b85e1e5b3329a784 2024-12-02T17:17:32,061 DEBUG [M:0;3b3375d1a925:41221 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eacd31949c7e40df870e56f53642cd9a is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1733159825141/Put/seqid=0 2024-12-02T17:17:32,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742578_1754 (size=45347) 2024-12-02T17:17:32,470 INFO [M:0;3b3375d1a925:41221 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=806.34 KB at sequenceid=2290 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eacd31949c7e40df870e56f53642cd9a 2024-12-02T17:17:32,474 INFO [M:0;3b3375d1a925:41221 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eacd31949c7e40df870e56f53642cd9a 2024-12-02T17:17:32,493 DEBUG [M:0;3b3375d1a925:41221 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b99f9cadedb44897964c2bdcb990b9bc is 69, key is 3b3375d1a925,40723,1733159666211/rs:state/1733159668652/Put/seqid=0 2024-12-02T17:17:32,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073742579_1755 (size=5156) 2024-12-02T17:17:32,897 INFO [M:0;3b3375d1a925:41221 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2290 (bloomFilter=true), to=hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b99f9cadedb44897964c2bdcb990b9bc 2024-12-02T17:17:32,900 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c399c93420344215b85e1e5b3329a784 as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c399c93420344215b85e1e5b3329a784 2024-12-02T17:17:32,903 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c399c93420344215b85e1e5b3329a784, entries=8, sequenceid=2290, filesize=5.5 K 2024-12-02T17:17:32,904 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/eacd31949c7e40df870e56f53642cd9a as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eacd31949c7e40df870e56f53642cd9a 2024-12-02T17:17:32,906 INFO [M:0;3b3375d1a925:41221 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eacd31949c7e40df870e56f53642cd9a 2024-12-02T17:17:32,906 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/eacd31949c7e40df870e56f53642cd9a, entries=170, sequenceid=2290, filesize=44.3 K 2024-12-02T17:17:32,907 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b99f9cadedb44897964c2bdcb990b9bc as hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b99f9cadedb44897964c2bdcb990b9bc 2024-12-02T17:17:32,910 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33791/user/jenkins/test-data/b9521639-586c-974c-1eb3-72a70c881fe0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b99f9cadedb44897964c2bdcb990b9bc, entries=1, sequenceid=2290, filesize=5.0 K 2024-12-02T17:17:32,910 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(3140): Finished flush of dataSize ~806.89 KB/826258, heapSize ~991.02 KB/1014800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2290, compaction requested=false 2024-12-02T17:17:32,913 INFO [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T17:17:32,913 DEBUG [M:0;3b3375d1a925:41221 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733159851617Disabling compacts and flushes for region at 1733159851617Disabling writes for close at 1733159851617Obtaining lock to block concurrent updates at 1733159851617Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733159851617Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=826258, getHeapSize=1015040, getOffHeapSize=0, getCellsCount=2485 at 1733159851617Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733159851618 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733159851618Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733159851631 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733159851631Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733159852038 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733159852061 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733159852061Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733159852475 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733159852492 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733159852492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33efe8b7: reopening flushed file at 1733159852900 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e11e7e1: reopening flushed file at 1733159852903 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4769d04f: reopening flushed file at 1733159852906 (+3 ms)Finished flush of dataSize ~806.89 KB/826258, heapSize ~991.02 KB/1014800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2290, compaction requested=false at 1733159852910 (+4 ms)Writing region close event to WAL at 1733159852913 (+3 ms)Closed at 1733159852913 2024-12-02T17:17:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36507 is added to blk_1073741830_1006 (size=976307) 2024-12-02T17:17:32,916 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T17:17:32,916 INFO [M:0;3b3375d1a925:41221 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T17:17:32,916 INFO [M:0;3b3375d1a925:41221 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41221 2024-12-02T17:17:32,916 INFO [M:0;3b3375d1a925:41221 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T17:17:33,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:17:33,018 INFO [M:0;3b3375d1a925:41221 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T17:17:33,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41221-0x1009bfbc5620000, quorum=127.0.0.1:58420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T17:17:33,023 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@250cb724{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T17:17:33,025 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@69edd336{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T17:17:33,025 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T17:17:33,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47c7fd6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T17:17:33,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/hadoop.log.dir/,STOPPED} 2024-12-02T17:17:33,028 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T17:17:33,028 WARN [BP-411259783-172.17.0.3-1733159661250 heartbeating to localhost/127.0.0.1:33791 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T17:17:33,029 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T17:17:33,029 WARN [BP-411259783-172.17.0.3-1733159661250 heartbeating to localhost/127.0.0.1:33791 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-411259783-172.17.0.3-1733159661250 (Datanode Uuid 1be79f1d-8f15-4d6f-a492-629389f9b831) service to localhost/127.0.0.1:33791 2024-12-02T17:17:33,031 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/data/data1/current/BP-411259783-172.17.0.3-1733159661250 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:17:33,031 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/cluster_d381d31b-14b8-0aff-b32b-35cbe2409ac5/data/data2/current/BP-411259783-172.17.0.3-1733159661250 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T17:17:33,032 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T17:17:33,040 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T17:17:33,040 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T17:17:33,040 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T17:17:33,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T17:17:33,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3665801d-7153-0ee1-867d-db41b5d768b4/hadoop.log.dir/,STOPPED} 2024-12-02T17:17:33,058 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T17:17:33,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down